fix(transform): suspend on yields in nested generator loop headers

changelog.d/10537-generator-loop-header-yield.md

@@ -0,0 +1,37 @@
+### Fixed
+
+- Generators now suspend on a `yield` in a loop header at any nesting depth
+  (#10419). A `yield` in a `while`/`do…while` condition or a `for`
+  condition/update was split into resume states only when the loop was a direct
+  statement of the generator body (#5933). Inside `if`/`else`, `try`/`catch`/
+  `finally`, a `switch` case, a label, or another loop, the residual yield never
+  suspended: `[...g()]` was empty, a sent-value loop such as
+  `while ((v = yield n) !== "stop")` never terminated, and `.return()`/`.throw()`
+  found a finished generator. Minifiers emit exactly this shape — lru-cache
+  11.5.2's `dist/esm/node/index.min.js` iterators (`keys`, `values`, `entries`,
+  `rkeys`, `forEach`, `for…of`, `dump`) returned nothing and `clear()` skipped
+  disposing entries.
+
+  Root cause: the linearizer only descends into a compound statement when
+  `body_contains_yield` reports a suspend point, and that check inspected loop
+  bodies but never loop headers, so the `if`/`try`/`switch`/label/loop around a
+  header-yield loop was emitted inline and the loop's per-iteration header arms
+  never ran. `body_contains_yield` now checks while/do-while conditions and for
+  conditions/updates. Three header positions the #5933 arms never covered, even
+  at top level, are fixed alongside: a for-init's own top-level yield
+  (`for (let t = yield x; …)`, `for (yield x; …)`, and an async function's
+  `for (let x = await p; …)` after the await→yield rewrite) is hoisted ahead of
+  the loop; a yielding update the header arm keeps in place (a `continue` inside
+  `try`/`finally`) is linearized in the loop's update state; and in an
+  `async function*` a header yield's operand is awaited like every other yield
+  operand (a `yield promise` in a loop condition delivered the promise itself).
+
+  Validation: `test_gap_10419_generator_loop_header_yield` (5 header positions ×
+  12 containers, sent values, `return()`/`throw()` mid-loop, labeled continue,
+  async generators with `for await`, plain-async for-init await) matches Node
+  byte-for-byte and differs on the pre-fix compiler; `perry-transform` unit tests
+  cover detection, the init hoist, residual-yield-free output and the async
+  operand await. lru-cache 11.5.2's default entry now matches Node on the audit
+  script (0 diff lines, was 14). Emitted LLVM IR for generators without header
+  yields (and for top-level header yields) is byte-identical to before, so their
+  instruction counts are unchanged.

crates/perry-transform/src/generator/break_continue.rs

@@ -323,19 +323,45 @@ pub fn body_contains_yield(stmts: &[Stmt]) -> bool {
                     }
                 }
             }
-            Stmt::While { body, .. } if body_contains_yield(body) => {
+            // A yield in a loop HEADER (while / do-while condition, for
+            // condition / update) suspends too (#10419). The linearizer's
+            // header arms split such a loop into per-iteration states, but
+            // they only run if every enclosing `if` / `try` / `switch` /
+            // label / loop is linearized as well — which is decided here.
+            // Checking only loop bodies left `if (n) while ((yield t), t > 0)`
+            // emitted inline: its residual `Expr::Yield` never suspended and
+            // the generator yielded nothing (minified lru-cache iterators).
+            Stmt::While { condition, body }
+                if super::hoist_yields::expr_contains_yield(condition)
+                    || body_contains_yield(body) =>
+            {
                 return true;
             }
             // A yield buried in a do-while or labeled loop must still be seen
             // by the enclosing construct's linearization (#1824), otherwise it
             // is never split into resume states.
-            Stmt::DoWhile { body, .. } if body_contains_yield(body) => {
+            Stmt::DoWhile { body, condition }
+                if super::hoist_yields::expr_contains_yield(condition)
+                    || body_contains_yield(body) =>
+            {
                 return true;
             }
             Stmt::Labeled { body, .. } if body_contains_yield(std::slice::from_ref(&**body)) => {
                 return true;
             }
-            Stmt::For { body, .. } if body_contains_yield(body) => {
+            Stmt::For {
+                condition,
+                update,
+                body,
+                ..
+            } if condition
+                .as_ref()
+                .is_some_and(super::hoist_yields::expr_contains_yield)
+                || update
+                    .as_ref()
+                    .is_some_and(super::hoist_yields::expr_contains_yield)
+                || body_contains_yield(body) =>
+            {
                 return true;
             }
             Stmt::Try {

crates/perry-transform/src/generator/hoist_yields.rs

@@ -101,11 +101,24 @@ fn hoist_yields_in_stmt(mut stmt: Stmt, next_id: &mut LocalId, hoisted: &mut Vec
             // of the loop. Condition/update are per-iteration — left in place
             // for the linearizer's For arm (matching the await pass's
             // single-hoist approximation for loop condition/update).
+            //
+            // The init's OWN top-level yield must be hoisted too (#10419):
+            // `for (let t = yield x; …)` / `for (yield x; …)` sit in a slot no
+            // linearizer arm splits, so the residual yield never suspended.
+            // The statement-level walk keeps a top-level yield in place, so
+            // hoist the init expression fully instead.
             if let Some(i) = init {
-                let mut inner = Vec::new();
-                let replaced = hoist_yields_in_stmt((**i).clone(), next_id, &mut inner);
-                hoisted.extend(inner);
-                **i = replaced;
+                match i.as_mut() {
+                    Stmt::Let { init: Some(e), .. } | Stmt::Expr(e) => {
+                        hoist_yields_in_expr_full(e, next_id, hoisted);
+                    }
+                    _ => {
+                        let mut inner = Vec::new();
+                        let replaced = hoist_yields_in_stmt((**i).clone(), next_id, &mut inner);
+                        hoisted.extend(inner);
+                        **i = replaced;
+                    }
+                }
             }
             hoist_yields_in_stmts(body, next_id);
         }

crates/perry-transform/src/generator/linearize.rs

@@ -412,6 +412,21 @@ pub struct FinallyRoute {
     pub completion_check_state: Option<u32>,
 }
 
+/// Normalize statements built from a loop HEADER (a condition or update the
+/// header arms move into the loop body / update state) into the statement
+/// shapes this linearizer splits. `hoist_yields_in_stmts` and — for an async
+/// generator — `await_async_generator_yield_operands` ran over the function
+/// body before linearization, but neither descends into loop headers, so a
+/// header yield reaches this point un-hoisted and, in an `async function*`,
+/// with its operand never awaited (spec `AsyncGeneratorYield(? Await(v))`:
+/// `yield promise` in a loop condition delivered the promise itself, #10419).
+fn normalize_loop_header_stmts(stmts: &mut Vec<Stmt>, next_local_id: &mut u32) {
+    hoist_yields_in_stmts(stmts, next_local_id);
+    if linearize_async_generator() {
+        super::lower::await_async_generator_yield_operands(stmts, next_local_id);
+    }
+}
+
 /// Linearize the generator body into a sequence of states.
 /// Splits at yield points and handles for-loops with yields.
 pub fn linearize_body(
@@ -694,7 +709,7 @@ pub fn linearize_body(
                     mutable: true,
                     init: Some(condition.clone()),
                 }];
-                hoist_yields_in_stmts(&mut prefix, next_local_id);
+                normalize_loop_header_stmts(&mut prefix, next_local_id);
                 prefix.push(Stmt::If {
                     condition: Expr::Unary {
                         op: UnaryOp::Not,
@@ -799,7 +814,7 @@ pub fn linearize_body(
                         mutable: true,
                         init: condition.clone(),
                     }];
-                    hoist_yields_in_stmts(&mut prefix, next_local_id);
+                    normalize_loop_header_stmts(&mut prefix, next_local_id);
                     new_body.append(&mut prefix);
                     new_body.push(Stmt::If {
                         condition: Expr::Unary {
@@ -813,7 +828,7 @@ pub fn linearize_body(
                 let mut taken_body = body.clone();
                 if upd_yields {
                     let mut upd_stmts = vec![Stmt::Expr(update.clone().unwrap())];
-                    hoist_yields_in_stmts(&mut upd_stmts, next_local_id);
+                    normalize_loop_header_stmts(&mut upd_stmts, next_local_id);
                     prefix_loop_continues(&mut taken_body, &upd_stmts);
                     new_body.append(&mut taken_body);
                     new_body.extend(upd_stmts);
@@ -839,13 +854,21 @@ pub fn linearize_body(
                 );
             }
 
-            // For-loop containing yield(s)
+            // For-loop containing yield(s) — in the body, or in an update the
+            // header arm above left in place (a `continue` inside
+            // try/finally must run the finally BEFORE the update, which the
+            // move-to-body-end rewrite cannot express). The update is then
+            // linearized in its own `continue`-target state below (#10419).
             Stmt::For {
                 init,
                 condition,
                 update,
                 body,
-            } if body_contains_yield(body) => {
+            } if body_contains_yield(body)
+                || update
+                    .as_ref()
+                    .is_some_and(super::hoist_yields::expr_contains_yield) =>
+            {
                 // State N: pre-loop code + init, goto condition check
                 let init_state = *state_num;
                 *state_num += 1;
@@ -954,22 +977,42 @@ pub fn linearize_body(
                 // residual, and (depending on guard placement) loop forever
                 // on the same iteration.
                 let update_state = *state_num;
-                *state_num += 1;
-                let mut update_body: Vec<Stmt> = Vec::new();
-                if let Some(upd) = update {
-                    update_body.push(Stmt::Expr(upd.clone()));
-                }
 
                 // Push tail_state pointing at update_state.
                 states.push(State {
                     num: tail_state,
                     body: tail_body,
                     exit: StateExit::Goto(update_state),
                 });
-                // Push update_state pointing at cond_state.
+                // Update statements, then the state that jumps back to
+                // cond_state. A yield in the update is split into its own
+                // states starting AT `update_state` (the first state pushed
+                // takes that number); without one, the final push below IS
+                // `update_state`, exactly as before.
+                if let Some(upd) = update {
+                    let mut upd_stmts = vec![Stmt::Expr(upd.clone())];
+                    if super::hoist_yields::expr_contains_yield(upd) {
+                        normalize_loop_header_stmts(&mut upd_stmts, next_local_id);
+                        linearize_body(
+                            &upd_stmts,
+                            states,
+                            current,
+                            state_num,
+                            state_id,
+                            next_local_id,
+                            sent_id,
+                            catches,
+                            finallys,
+                        );
+                    } else {
+                        current.append(&mut upd_stmts);
+                    }
+                }
+                let update_tail_state = *state_num;
+                *state_num += 1;
                 states.push(State {
-                    num: update_state,
-                    body: update_body,
+                    num: update_tail_state,
+                    body: std::mem::take(current),
                     exit: StateExit::Goto(cond_state),
                 });