From e7d87b664ace9d8b9deb2012e1f097609200e459 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Mon, 14 Sep 2026 20:22:25 +0200 Subject: [PATCH] Remove allocation from `mbe::quoted::parse` --- compiler/rustc_expand/src/lib.rs | 1 + compiler/rustc_expand/src/mbe/quoted.rs | 220 ++++++++++++------------ 2 files changed, 111 insertions(+), 110 deletions(-) diff --git a/compiler/rustc_expand/src/lib.rs b/compiler/rustc_expand/src/lib.rs index c81d9eb12aa48..9e23fb87e3b8b 100644 --- a/compiler/rustc_expand/src/lib.rs +++ b/compiler/rustc_expand/src/lib.rs @@ -8,6 +8,7 @@ #![feature(proc_macro_internals)] #![feature(try_blocks)] #![feature(yeet_expr)] +#![feature(gen_blocks)] // tidy-alphabetical-end mod build; diff --git a/compiler/rustc_expand/src/mbe/quoted.rs b/compiler/rustc_expand/src/mbe/quoted.rs index 2779291abf361..a2acbd95be32a 100644 --- a/compiler/rustc_expand/src/mbe/quoted.rs +++ b/compiler/rustc_expand/src/mbe/quoted.rs @@ -38,8 +38,8 @@ impl RulePart { } } -/// Takes a `tokenstream::TokenStream` and returns a `Vec`. Specifically, this -/// takes a generic `TokenStream`, such as is used in the rest of the compiler, and returns a +/// Takes a `tokenstream::TokenStream` and returns an iterator of `self::TokenTree`. Specifically, +/// this takes a generic `TokenStream`, such as is used in the rest of the compiler, and returns a /// collection of `TokenTree` for use in parsing a macro. /// /// # Parameters @@ -57,7 +57,7 @@ impl RulePart { /// /// # Returns /// -/// A collection of `self::TokenTree`. There may also be some errors emitted to `sess`. +/// An iterator of `self::TokenTree`. There may also be some errors emitted to `sess`. fn parse( input: &tokenstream::TokenStream, part: RulePart, @@ -65,119 +65,118 @@ fn parse( node_id: NodeId, features: &Features, edition: Edition, -) -> Vec { - // Will contain the final collection of `self::TokenTree` - let mut result = Vec::new(); - - // For each token tree in `input`, parse the token into a `self::TokenTree`, consuming - // additional trees if need be. - let mut iter = input.iter(); - while let Some(tree) = iter.next() { - // Given the parsed tree, if there is a metavar and we are expecting matchers, actually - // parse out the matcher (i.e., in `$id:ident` this would parse the `:` and `ident`). - let tree = parse_tree(tree, &mut iter, part, sess, node_id, features, edition); - - if part.is_body() { - // No matchers allowed, nothing to process here - result.push(tree); - continue; - } - - let TokenTree::MetaVar(start_sp, ident) = tree else { - // Not a metavariable, just return the tree - result.push(tree); - continue; - }; - - let fallback_metavar_decl = - |span| TokenTree::MetaVarDecl { span, name: ident, kind: NonterminalKind::TT }; - // Emit a missing-fragment diagnostic and return a `TokenTree` fallback so parsing can - // continue. - let missing_fragment_specifier = |span, add_span| { - sess.dcx().emit_err(diagnostics::MissingFragmentSpecifier { - span, - add_span, - valid: VALID_FRAGMENT_NAMES_MSG, - }); - fallback_metavar_decl(span) - }; - - // Not consuming the next token immediately, as it may not be a colon - if let Some(peek) = iter.peek() - && let tokenstream::TokenTree::Token(token, _spacing) = peek - && let Token { kind: token::Colon, span: colon_span } = token - { - // Next token is a colon; consume it - iter.next(); - - // It's ok to consume the next tree no matter how, - // since if it's not a token then it will be an invalid declaration. - let Some(tokenstream::TokenTree::Token(token, _)) = iter.next() else { - // Invalid, return a nice source location as `var:` - result.push(missing_fragment_specifier( - colon_span.with_lo(start_sp.lo()), - colon_span.shrink_to_hi(), - )); +) -> impl Iterator { + gen move { + // For each token tree in `input`, parse the token into a `self::TokenTree`, consuming + // additional trees if need be. + let mut iter = input.iter(); + while let Some(tree) = iter.next() { + // Given the parsed tree, if there is a metavar and we are expecting matchers, actually + // parse out the matcher (i.e., in `$id:ident` this would parse the `:` and `ident`). + let tree = parse_tree(tree, &mut iter, part, sess, node_id, features, edition); + + if part.is_body() { + // No matchers allowed, nothing to process here + yield tree; continue; - }; + } - let Some((fragment, _)) = token.ident() else { - // No identifier for the fragment specifier; - if token.kind == token::Dollar - && iter.peek().is_some_and(|next| { - matches!( - next, - tokenstream::TokenTree::Token(next_token, _) - if next_token.ident().is_some() - ) - }) - { - let mut err = - sess.dcx().struct_span_err(token.span, "missing fragment specifier"); - err.note("fragment specifiers must be provided"); - err.help(VALID_FRAGMENT_NAMES_MSG); - err.span_suggestion_verbose( - token.span, - "fragment specifiers should not be prefixed with `$`", - "", - Applicability::MaybeIncorrect, - ); - err.emit(); - result.push(fallback_metavar_decl(token.span)); - } else { - result.push(missing_fragment_specifier(token.span, token.span.shrink_to_hi())); - } + let TokenTree::MetaVar(start_sp, ident) = tree else { + // Not a metavariable, just return the tree + yield tree; continue; }; - let span = token.span.with_lo(start_sp.lo()); - let edition = || { - // FIXME(#85708) - once we properly decode a foreign - // crate's `SyntaxContext::root`, then we can replace - // this with just `span.edition()`. A - // `SyntaxContext::root()` from the current crate will - // have the edition of the current crate, and a - // `SyntaxContext::root()` from a foreign crate will - // have the edition of that crate (which we manually - // retrieve via the `edition` parameter). - if !span.from_expansion() { edition } else { span.edition() } - }; - let kind = NonterminalKind::from_symbol(fragment.name, edition).unwrap_or_else(|| { - sess.dcx().emit_err(diagnostics::InvalidFragmentSpecifier { + let fallback_metavar_decl = + |span| TokenTree::MetaVarDecl { span, name: ident, kind: NonterminalKind::TT }; + // Emit a missing-fragment diagnostic and return a `TokenTree` fallback so parsing can + // continue. + let missing_fragment_specifier = |span, add_span| { + sess.dcx().emit_err(diagnostics::MissingFragmentSpecifier { span, - fragment, - help: VALID_FRAGMENT_NAMES_MSG, + add_span, + valid: VALID_FRAGMENT_NAMES_MSG, }); - NonterminalKind::TT - }); - result.push(TokenTree::MetaVarDecl { span, name: ident, kind }); - } else { - // Whether it's none or some other tree, it doesn't belong to - // the current meta variable, returning the original span. - result.push(missing_fragment_specifier(start_sp, start_sp.shrink_to_hi())); + fallback_metavar_decl(span) + }; + + // Not consuming the next token immediately, as it may not be a colon + if let Some(peek) = iter.peek() + && let tokenstream::TokenTree::Token(token, _spacing) = peek + && let Token { kind: token::Colon, span: colon_span } = token + { + // Next token is a colon; consume it + iter.next(); + + // It's ok to consume the next tree no matter how, + // since if it's not a token then it will be an invalid declaration. + let Some(tokenstream::TokenTree::Token(token, _)) = iter.next() else { + // Invalid, return a nice source location as `var:` + yield missing_fragment_specifier( + colon_span.with_lo(start_sp.lo()), + colon_span.shrink_to_hi(), + ); + continue; + }; + + let Some((fragment, _)) = token.ident() else { + // No identifier for the fragment specifier; + if token.kind == token::Dollar + && iter.peek().is_some_and(|next| { + matches!( + next, + tokenstream::TokenTree::Token(next_token, _) + if next_token.ident().is_some() + ) + }) + { + let mut err = + sess.dcx().struct_span_err(token.span, "missing fragment specifier"); + err.note("fragment specifiers must be provided"); + err.help(VALID_FRAGMENT_NAMES_MSG); + err.span_suggestion_verbose( + token.span, + "fragment specifiers should not be prefixed with `$`", + "", + Applicability::MaybeIncorrect, + ); + err.emit(); + yield fallback_metavar_decl(token.span); + } else { + yield missing_fragment_specifier(token.span, token.span.shrink_to_hi()); + } + continue; + }; + + let span = token.span.with_lo(start_sp.lo()); + let edition = || { + // FIXME(#85708) - once we properly decode a foreign + // crate's `SyntaxContext::root`, then we can replace + // this with just `span.edition()`. A + // `SyntaxContext::root()` from the current crate will + // have the edition of the current crate, and a + // `SyntaxContext::root()` from a foreign crate will + // have the edition of that crate (which we manually + // retrieve via the `edition` parameter). + if !span.from_expansion() { edition } else { span.edition() } + }; + let kind = + NonterminalKind::from_symbol(fragment.name, edition).unwrap_or_else(|| { + sess.dcx().emit_err(diagnostics::InvalidFragmentSpecifier { + span, + fragment, + help: VALID_FRAGMENT_NAMES_MSG, + }); + NonterminalKind::TT + }); + yield TokenTree::MetaVarDecl { span, name: ident, kind }; + } else { + // Whether it's none or some other tree, it doesn't belong to + // the current meta variable, returning the original span. + yield missing_fragment_specifier(start_sp, start_sp.shrink_to_hi()); + } } } - result } /// Takes a `tokenstream::TokenTree` and returns a `self::TokenTree`. Like `parse`, but for a @@ -192,7 +191,7 @@ pub(super) fn parse_one_tt( edition: Edition, ) -> TokenTree { parse(&tokenstream::TokenStream::new(vec![input]), part, sess, node_id, features, edition) - .pop() + .last() .unwrap() } @@ -309,7 +308,8 @@ fn parse_tree<'a>( // If we didn't find a metavar expression above, then we must have a // repetition sequence in the macro (e.g. `$(pat)*`). Parse the // contents of the sequence itself - let sequence = parse(tts, part, sess, node_id, features, edition); + let sequence = + parse(tts, part, sess, node_id, features, edition).collect::>(); // Get the Kleene operator and optional separator let (separator, kleene) = parse_sep_and_kleene_op(iter, delim_span.entire(), sess); @@ -371,7 +371,7 @@ fn parse_tree<'a>( &tokenstream::TokenTree::Delimited(span, spacing, delim, ref tts) => TokenTree::Delimited( span, spacing, - Delimited { delim, tts: parse(tts, part, sess, node_id, features, edition) }, + Delimited { delim, tts: parse(tts, part, sess, node_id, features, edition).collect() }, ), } }