Skip to content

Commit 46d8c4b

Browse files
Aaron1011Aaron1011
committed
Fix recursive nonterminal expansion during pretty-print/reparse check
Makes progress towards #43081 In PR #73084, we started recursively expanded nonterminals during the pretty-print/reparse check, allowing them to be properly compared against the reparsed tokenstream. Unfortunately, the recursive logic in that PR only handles the case where a nonterminal appears inside a `TokenTree::Delimited`. If a nonterminal appears directly in the expanded tokens of another nonterminal, the inner nonterminal will not be expanded. This PR fixes the recursive expansion of nonterminals, ensuring that they are expanded wherever they occur.
1 parent d62d3f7 commit 46d8c4b

File tree

3 files changed

+120
-23
lines changed

3 files changed

+120
-23
lines changed

‎compiler/rustc_parse/src/lib.rs

+34-23
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88

99
use rustc_ast as ast;
1010
use rustc_ast::token::{self, Nonterminal, Token, TokenKind};
11-
use rustc_ast::tokenstream::{self, Spacing, TokenStream, TokenTree};
11+
use rustc_ast::tokenstream::{self, TokenStream, TokenTree};
1212
use rustc_ast_pretty::pprust;
1313
use rustc_data_structures::sync::Lrc;
1414
use rustc_errors::{Diagnostic, FatalError, Level, PResult};
@@ -435,31 +435,42 @@ pub fn tokenstream_probably_equal_for_proc_macro(
435435
token_trees.into_iter()
436436
}
437437

438-
let expand_nt = |tree: TokenTree| {
439-
if let TokenTree::Token(Token { kind: TokenKind::Interpolated(nt), span }) = &tree {
440-
// When checking tokenstreams for 'probable equality', we are comparing
441-
// a captured (from parsing) `TokenStream` to a reparsed tokenstream.
442-
// The reparsed Tokenstream will never have `None`-delimited groups,
443-
// since they are only ever inserted as a result of macro expansion.
444-
// Therefore, inserting a `None`-delimtied group here (when we
445-
// convert a nested `Nonterminal` to a tokenstream) would cause
446-
// a mismatch with the reparsed tokenstream.
447-
//
448-
// Note that we currently do not handle the case where the
449-
// reparsed stream has a `Parenthesis`-delimited group
450-
// inserted. This will cause a spurious mismatch:
451-
// issue #75734 tracks resolving this.
452-
nt_to_tokenstream(nt, sess, *span).into_trees()
453-
} else {
454-
TokenStream::new(vec![(tree, Spacing::Alone)]).into_trees()
455-
}
456-
};
438+
fn expand_token(tree: TokenTree, sess: &ParseSess) -> impl Iterator<Item = TokenTree> {
439+
// When checking tokenstreams for 'probable equality', we are comparing
440+
// a captured (from parsing) `TokenStream` to a reparsed tokenstream.
441+
// The reparsed Tokenstream will never have `None`-delimited groups,
442+
// since they are only ever inserted as a result of macro expansion.
443+
// Therefore, inserting a `None`-delimtied group here (when we
444+
// convert a nested `Nonterminal` to a tokenstream) would cause
445+
// a mismatch with the reparsed tokenstream.
446+
//
447+
// Note that we currently do not handle the case where the
448+
// reparsed stream has a `Parenthesis`-delimited group
449+
// inserted. This will cause a spurious mismatch:
450+
// issue #75734 tracks resolving this.
451+
452+
let expanded: SmallVec<[_; 1]> =
453+
if let TokenTree::Token(Token { kind: TokenKind::Interpolated(nt), span }) = &tree {
454+
nt_to_tokenstream(nt, sess, *span)
455+
.into_trees()
456+
.flat_map(|t| expand_token(t, sess))
457+
.collect()
458+
} else {
459+
// Filter before and after breaking tokens,
460+
// since we may want to ignore both glued and unglued tokens.
461+
std::iter::once(tree)
462+
.filter(semantic_tree)
463+
.flat_map(break_tokens)
464+
.filter(semantic_tree)
465+
.collect()
466+
};
467+
expanded.into_iter()
468+
}
457469

458470
// Break tokens after we expand any nonterminals, so that we break tokens
459471
// that are produced as a result of nonterminal expansion.
460-
let tokens = tokens.trees().filter(semantic_tree).flat_map(expand_nt).flat_map(break_tokens);
461-
let reparsed_tokens =
462-
reparsed_tokens.trees().filter(semantic_tree).flat_map(expand_nt).flat_map(break_tokens);
472+
let tokens = tokens.trees().flat_map(|t| expand_token(t, sess));
473+
let reparsed_tokens = reparsed_tokens.trees().flat_map(|t| expand_token(t, sess));
463474

464475
tokens.eq_by(reparsed_tokens, |t, rt| tokentree_probably_equal_for_proc_macro(&t, &rt, sess))
465476
}

‎src/test/ui/proc-macro/nested-nonterminal-tokens.rs

+26
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
// check-pass
2+
// edition:2018
3+
// compile-flags: -Z span-debug
4+
// aux-build:test-macros.rs
5+
6+
// Tests that we properly pass tokens to proc-macro when nested
7+
// nonterminals are involved.
8+
9+
#![no_std] // Don't load unnecessary hygiene information from std
10+
extern crate std;
11+
12+
#[macro_use]
13+
extern crate test_macros;
14+
15+
16+
macro_rules! wrap {
17+
(first, $e:expr) => { wrap!(second, $e + 1) };
18+
(second, $e:expr) => { wrap!(third, $e + 2) };
19+
(third, $e:expr) => {
20+
print_bang!($e + 3);
21+
};
22+
}
23+
24+
fn main() {
25+
let _ = wrap!(first, 0);
26+
}

‎src/test/ui/proc-macro/nested-nonterminal-tokens.stdout

+60
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
PRINT-BANG INPUT (DISPLAY): 0 + 1 + 2 + 3
2+
PRINT-BANG INPUT (DEBUG): TokenStream [
3+
Group {
4+
delimiter: None,
5+
stream: TokenStream [
6+
Group {
7+
delimiter: None,
8+
stream: TokenStream [
9+
Group {
10+
delimiter: None,
11+
stream: TokenStream [
12+
Literal {
13+
kind: Integer,
14+
symbol: "0",
15+
suffix: None,
16+
span: $DIR/nested-nonterminal-tokens.rs:25:26: 25:27 (#0),
17+
},
18+
],
19+
span: $DIR/nested-nonterminal-tokens.rs:17:41: 17:43 (#4),
20+
},
21+
Punct {
22+
ch: '+',
23+
spacing: Alone,
24+
span: $DIR/nested-nonterminal-tokens.rs:17:44: 17:45 (#4),
25+
},
26+
Literal {
27+
kind: Integer,
28+
symbol: "1",
29+
suffix: None,
30+
span: $DIR/nested-nonterminal-tokens.rs:17:46: 17:47 (#4),
31+
},
32+
],
33+
span: $DIR/nested-nonterminal-tokens.rs:18:41: 18:43 (#5),
34+
},
35+
Punct {
36+
ch: '+',
37+
spacing: Alone,
38+
span: $DIR/nested-nonterminal-tokens.rs:18:44: 18:45 (#5),
39+
},
40+
Literal {
41+
kind: Integer,
42+
symbol: "2",
43+
suffix: None,
44+
span: $DIR/nested-nonterminal-tokens.rs:18:46: 18:47 (#5),
45+
},
46+
],
47+
span: $DIR/nested-nonterminal-tokens.rs:20:21: 20:23 (#6),
48+
},
49+
Punct {
50+
ch: '+',
51+
spacing: Alone,
52+
span: $DIR/nested-nonterminal-tokens.rs:20:24: 20:25 (#6),
53+
},
54+
Literal {
55+
kind: Integer,
56+
symbol: "3",
57+
suffix: None,
58+
span: $DIR/nested-nonterminal-tokens.rs:20:26: 20:27 (#6),
59+
},
60+
]

0 commit comments

Comments
 (0)