Skip to content

Remove allocation from mbe::quoted::parse #162777

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
Kobzol wants to merge 1 commit into from
+111 −110
Closed

Remove allocation from mbe::quoted::parse #162777

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions compiler/rustc_expand/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
#![feature(proc_macro_internals)]
#![feature(try_blocks)]
#![feature(yeet_expr)]
#![feature(gen_blocks)]
// tidy-alphabetical-end

mod build;
Expand Down
220 changes: 110 additions & 110 deletions compiler/rustc_expand/src/mbe/quoted.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,8 +38,8 @@ impl RulePart {
}
}

/// Takes a `tokenstream::TokenStream` and returns a `Vec<self::TokenTree>`. Specifically, this
/// takes a generic `TokenStream`, such as is used in the rest of the compiler, and returns a
/// Takes a `tokenstream::TokenStream` and returns an iterator of `self::TokenTree`. Specifically,
/// this takes a generic `TokenStream`, such as is used in the rest of the compiler, and returns a
/// collection of `TokenTree` for use in parsing a macro.
///
/// # Parameters
Expand All @@ -57,127 +57,126 @@ impl RulePart {
///
/// # Returns
///
/// A collection of `self::TokenTree`. There may also be some errors emitted to `sess`.
/// An iterator of `self::TokenTree`. There may also be some errors emitted to `sess`.
fn parse(
input: &tokenstream::TokenStream,
part: RulePart,
sess: &Session,
node_id: NodeId,
features: &Features,
edition: Edition,
) -> Vec<TokenTree> {
// Will contain the final collection of `self::TokenTree`
let mut result = Vec::new();

// For each token tree in `input`, parse the token into a `self::TokenTree`, consuming
// additional trees if need be.
let mut iter = input.iter();
while let Some(tree) = iter.next() {
// Given the parsed tree, if there is a metavar and we are expecting matchers, actually
// parse out the matcher (i.e., in `$id:ident` this would parse the `:` and `ident`).
let tree = parse_tree(tree, &mut iter, part, sess, node_id, features, edition);

if part.is_body() {
// No matchers allowed, nothing to process here
result.push(tree);
continue;
}

let TokenTree::MetaVar(start_sp, ident) = tree else {
// Not a metavariable, just return the tree
result.push(tree);
continue;
};

let fallback_metavar_decl =
|span| TokenTree::MetaVarDecl { span, name: ident, kind: NonterminalKind::TT };
// Emit a missing-fragment diagnostic and return a `TokenTree` fallback so parsing can
// continue.
let missing_fragment_specifier = |span, add_span| {
sess.dcx().emit_err(diagnostics::MissingFragmentSpecifier {
span,
add_span,
valid: VALID_FRAGMENT_NAMES_MSG,
});
fallback_metavar_decl(span)
};

// Not consuming the next token immediately, as it may not be a colon
if let Some(peek) = iter.peek()
&& let tokenstream::TokenTree::Token(token, _spacing) = peek
&& let Token { kind: token::Colon, span: colon_span } = token
{
// Next token is a colon; consume it
iter.next();

// It's ok to consume the next tree no matter how,
// since if it's not a token then it will be an invalid declaration.
let Some(tokenstream::TokenTree::Token(token, _)) = iter.next() else {
// Invalid, return a nice source location as `var:`
result.push(missing_fragment_specifier(
colon_span.with_lo(start_sp.lo()),
colon_span.shrink_to_hi(),
));
) -> impl Iterator<Item = TokenTree> {
gen move {
// For each token tree in `input`, parse the token into a `self::TokenTree`, consuming
// additional trees if need be.
let mut iter = input.iter();
while let Some(tree) = iter.next() {
// Given the parsed tree, if there is a metavar and we are expecting matchers, actually
// parse out the matcher (i.e., in `$id:ident` this would parse the `:` and `ident`).
let tree = parse_tree(tree, &mut iter, part, sess, node_id, features, edition);

if part.is_body() {
// No matchers allowed, nothing to process here
yield tree;
continue;
};
}

let Some((fragment, _)) = token.ident() else {
// No identifier for the fragment specifier;
if token.kind == token::Dollar
&& iter.peek().is_some_and(|next| {
matches!(
next,
tokenstream::TokenTree::Token(next_token, _)
if next_token.ident().is_some()
)
})
{
let mut err =
sess.dcx().struct_span_err(token.span, "missing fragment specifier");
err.note("fragment specifiers must be provided");
err.help(VALID_FRAGMENT_NAMES_MSG);
err.span_suggestion_verbose(
token.span,
"fragment specifiers should not be prefixed with `$`",
"",
Applicability::MaybeIncorrect,
);
err.emit();
result.push(fallback_metavar_decl(token.span));
} else {
result.push(missing_fragment_specifier(token.span, token.span.shrink_to_hi()));
}
let TokenTree::MetaVar(start_sp, ident) = tree else {
// Not a metavariable, just return the tree
yield tree;
continue;
};

let span = token.span.with_lo(start_sp.lo());
let edition = || {
// FIXME(#85708) - once we properly decode a foreign
// crate's `SyntaxContext::root`, then we can replace
// this with just `span.edition()`. A
// `SyntaxContext::root()` from the current crate will
// have the edition of the current crate, and a
// `SyntaxContext::root()` from a foreign crate will
// have the edition of that crate (which we manually
// retrieve via the `edition` parameter).
if !span.from_expansion() { edition } else { span.edition() }
};
let kind = NonterminalKind::from_symbol(fragment.name, edition).unwrap_or_else(|| {
sess.dcx().emit_err(diagnostics::InvalidFragmentSpecifier {
let fallback_metavar_decl =
|span| TokenTree::MetaVarDecl { span, name: ident, kind: NonterminalKind::TT };
// Emit a missing-fragment diagnostic and return a `TokenTree` fallback so parsing can
// continue.
let missing_fragment_specifier = |span, add_span| {
sess.dcx().emit_err(diagnostics::MissingFragmentSpecifier {
span,
fragment,
help: VALID_FRAGMENT_NAMES_MSG,
add_span,
valid: VALID_FRAGMENT_NAMES_MSG,
});
NonterminalKind::TT
});
result.push(TokenTree::MetaVarDecl { span, name: ident, kind });
} else {
// Whether it's none or some other tree, it doesn't belong to
// the current meta variable, returning the original span.
result.push(missing_fragment_specifier(start_sp, start_sp.shrink_to_hi()));
fallback_metavar_decl(span)
};

// Not consuming the next token immediately, as it may not be a colon
if let Some(peek) = iter.peek()
&& let tokenstream::TokenTree::Token(token, _spacing) = peek
&& let Token { kind: token::Colon, span: colon_span } = token
{
// Next token is a colon; consume it
iter.next();

// It's ok to consume the next tree no matter how,
// since if it's not a token then it will be an invalid declaration.
let Some(tokenstream::TokenTree::Token(token, _)) = iter.next() else {
// Invalid, return a nice source location as `var:`
yield missing_fragment_specifier(
colon_span.with_lo(start_sp.lo()),
colon_span.shrink_to_hi(),
);
continue;
};

let Some((fragment, _)) = token.ident() else {
// No identifier for the fragment specifier;
if token.kind == token::Dollar
&& iter.peek().is_some_and(|next| {
matches!(
next,
tokenstream::TokenTree::Token(next_token, _)
if next_token.ident().is_some()
)
})
{
let mut err =
sess.dcx().struct_span_err(token.span, "missing fragment specifier");
err.note("fragment specifiers must be provided");
err.help(VALID_FRAGMENT_NAMES_MSG);
err.span_suggestion_verbose(
token.span,
"fragment specifiers should not be prefixed with `$`",
"",
Applicability::MaybeIncorrect,
);
err.emit();
yield fallback_metavar_decl(token.span);
} else {
yield missing_fragment_specifier(token.span, token.span.shrink_to_hi());
}
continue;
};

let span = token.span.with_lo(start_sp.lo());
let edition = || {
// FIXME(#85708) - once we properly decode a foreign
// crate's `SyntaxContext::root`, then we can replace
// this with just `span.edition()`. A
// `SyntaxContext::root()` from the current crate will
// have the edition of the current crate, and a
// `SyntaxContext::root()` from a foreign crate will
// have the edition of that crate (which we manually
// retrieve via the `edition` parameter).
if !span.from_expansion() { edition } else { span.edition() }
};
let kind =
NonterminalKind::from_symbol(fragment.name, edition).unwrap_or_else(|| {
sess.dcx().emit_err(diagnostics::InvalidFragmentSpecifier {
span,
fragment,
help: VALID_FRAGMENT_NAMES_MSG,
});
NonterminalKind::TT
});
yield TokenTree::MetaVarDecl { span, name: ident, kind };
} else {
// Whether it's none or some other tree, it doesn't belong to
// the current meta variable, returning the original span.
yield missing_fragment_specifier(start_sp, start_sp.shrink_to_hi());
}
}
}
result
}

/// Takes a `tokenstream::TokenTree` and returns a `self::TokenTree`. Like `parse`, but for a
Expand All @@ -192,7 +191,7 @@ pub(super) fn parse_one_tt(
edition: Edition,
) -> TokenTree {
parse(&tokenstream::TokenStream::new(vec![input]), part, sess, node_id, features, edition)
.pop()
.last()
.unwrap()
}

Expand Down Expand Up @@ -309,7 +308,8 @@ fn parse_tree<'a>(
// If we didn't find a metavar expression above, then we must have a
// repetition sequence in the macro (e.g. `$(pat)*`). Parse the
// contents of the sequence itself
let sequence = parse(tts, part, sess, node_id, features, edition);
let sequence =
parse(tts, part, sess, node_id, features, edition).collect::<Vec<_>>();
// Get the Kleene operator and optional separator
let (separator, kleene) =
parse_sep_and_kleene_op(iter, delim_span.entire(), sess);
Expand Down Expand Up @@ -371,7 +371,7 @@ fn parse_tree<'a>(
&tokenstream::TokenTree::Delimited(span, spacing, delim, ref tts) => TokenTree::Delimited(
span,
spacing,
Delimited { delim, tts: parse(tts, part, sess, node_id, features, edition) },
Delimited { delim, tts: parse(tts, part, sess, node_id, features, edition).collect() },
),
}
}
Expand Down
Loading