Skip to content

Commit 9b9f8f0

Browse files
nnethercotennethercote
committed
Improve token stream pretty-printing.
As well as nicer output, this fixes several FIXMEs in `tests/ui/macros/stringify.rs`, where the current output is sub-optimal.
1 parent f54eb04 commit 9b9f8f0

File tree

61 files changed

+428
-339
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

61 files changed

+428
-339
lines changed

‎compiler/rustc_ast_pretty/src/lib.rs

+2
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@
55
#![deny(rustc::diagnostic_outside_of_impl)]
66
#![feature(associated_type_bounds)]
77
#![feature(box_patterns)]
8+
#![feature(if_let_guard)]
9+
#![feature(let_chains)]
810
#![feature(with_negative_coherence)]
911
#![recursion_limit = "256"]
1012

‎compiler/rustc_ast_pretty/src/pprust/state.rs

+125-25
Original file line numberDiff line numberDiff line change
@@ -146,47 +146,142 @@ pub fn print_crate<'a>(
146146
s.s.eof()
147147
}
148148

149-
/// Should two consecutive tokens be printed with a space between them?
149+
fn is_punct(tt: &TokenTree) -> bool {
150+
matches!(tt, TokenTree::Token(tok, _) if tok.is_punct())
151+
}
152+
153+
/// Should two consecutive token trees be printed with a space between them?
154+
///
155+
/// NOTE: should always be false if both token trees are punctuation, so that
156+
/// any old proc macro that parses pretty-printed code won't glue together
157+
/// tokens that shouldn't be glued.
150158
///
151159
/// Note: some old proc macros parse pretty-printed output, so changes here can
152160
/// break old code. For example:
153161
/// - #63896: `#[allow(unused,` must be printed rather than `#[allow(unused ,`
154162
/// - #73345: `#[allow(unused)] must be printed rather than `# [allow(unused)]
155163
///
156-
fn space_between(tt1: &TokenTree, tt2: &TokenTree) -> bool {
164+
fn space_between(prev: Option<&TokenTree>, tt1: &TokenTree, tt2: &TokenTree) -> bool {
157165
use token::*;
158166
use Delimiter::*;
159167
use TokenTree::Delimited as Del;
160168
use TokenTree::Token as Tok;
161169

162-
// Each match arm has one or more examples in comments. The default is to
163-
// insert space between adjacent tokens, except for the cases listed in
164-
// this match.
170+
// Each match arm has one or more examples in comments.
165171
match (tt1, tt2) {
166172
// No space after line doc comments.
167173
(Tok(Token { kind: DocComment(CommentKind::Line, ..), .. }, _), _) => false,
168174

169-
// `.` + ANYTHING: `x.y`, `tup.0`
170-
// `$` + ANYTHING: `$e`
171-
(Tok(Token { kind: Dot | Dollar, .. }, _), _) => false,
172-
173-
// ANYTHING + `,`: `foo,`
174-
// ANYTHING + `.`: `x.y`, `tup.0`
175-
// ANYTHING + `!`: `foo! { ... }`
176-
//
177-
// FIXME: Incorrect cases:
178-
// - Logical not: `x =! y`, `if! x { f(); }`
179-
// - Never type: `Fn() ->!`
180-
(_, Tok(Token { kind: Comma | Dot | Not, .. }, _)) => false,
175+
// `.` + NON-PUNCT: `x.y`, `tup.0`
176+
// `$` + NON-PUNCT: `$e`
177+
(Tok(Token { kind: Dot | Dollar, .. }, _), tt2) if !is_punct(tt2) => false,
178+
179+
// NON-PUNCT + `,`: `foo,`
180+
// NON-PUNCT + `;`: `x = 3;`, `[T; 3]`
181+
// NON-PUNCT + `.`: `x.y`, `tup.0`
182+
// NON-PUNCT + `:`: `'a: loop { ... }`, `x: u8`, `where T: U`,
183+
// `<Self as T>::x`, `Trait<'a>: Sized`, `X<Y<Z>>: Send`,
184+
// `let (a, b): (u32, u32);`
185+
(tt1, Tok(Token { kind: Comma | Semi | Dot | Colon, .. }, _)) if !is_punct(tt1) => false,
186+
187+
// ANYTHING-BUT-`,`|`:`|`mut`|`<` + `[`: `<expr>[1]`, `vec![]`, `#[attr]`,
188+
// `#![attr]`, but not `data: [T; 0]`, `f(a, [])`, `&mut [T]`,
189+
// `NonNull< [T] >`
190+
(Tok(Token { kind: Comma | Colon | Lt, .. }, _), Del(_, Bracket, _)) => true,
191+
(Tok(Token { kind: Ident(sym, is_raw), .. }, _), Del(_, Bracket, _))
192+
if *sym == kw::Mut && !is_raw =>
193+
{
194+
true
195+
}
196+
(Tok(_, _), Del(_, Bracket, _)) => false,
197+
198+
// IDENT|`fn`|`Self`|`pub` + `(`: `f(3)`, `fn(x: u8)`, `Self()`, `pub(crate)`,
199+
// but `let (a, b) = (1, 2)` needs a space after the `let`
200+
(Tok(Token { kind: Ident(sym, is_raw), span }, _), Del(_, Parenthesis, _))
201+
if !Ident::new(*sym, *span).is_reserved()
202+
|| *sym == kw::Fn
203+
|| *sym == kw::SelfUpper
204+
|| *sym == kw::Pub
205+
|| *is_raw =>
206+
{
207+
false
208+
}
209+
210+
// IDENT|`self`|`Self`|`$crate`|`crate`|`super` + `::`: `x::y`,
211+
// `Self::a`, `$crate::x`, `crate::x`, `super::x`, but
212+
// `if ::a::b() { ... }` needs a space after the `if`.
213+
(Tok(Token { kind: Ident(sym, is_raw), span }, _), Tok(Token { kind: ModSep, .. }, _))
214+
if !Ident::new(*sym, *span).is_reserved()
215+
|| sym.is_path_segment_keyword()
216+
|| *is_raw =>
217+
{
218+
false
219+
}
220+
221+
// `::` + IDENT: `foo::bar`
222+
// `::` + `{`: `use a::{b, c}`
223+
(
224+
Tok(Token { kind: ModSep, .. }, _),
225+
Tok(Token { kind: Ident(..), .. }, _) | Del(_, Brace, _),
226+
) => false,
227+
228+
// `impl` + `<`: `impl<T> Foo<T> { ... }`
229+
// `for` + `<`: `for<'a> fn()`
230+
(Tok(Token { kind: Ident(sym, is_raw), .. }, _), Tok(Token { kind: Lt, .. }, _))
231+
if (*sym == kw::Impl || *sym == kw::For) && !is_raw =>
232+
{
233+
false
234+
}
235+
236+
// `fn` + IDENT + `<`: `fn f<T>(t: T) { ... }`
237+
(Tok(Token { kind: Ident(..), .. }, _), Tok(Token { kind: Lt, .. }, _))
238+
if let Some(prev) = prev
239+
&& let Tok(Token { kind: Ident(sym, is_raw), .. }, _) = prev
240+
&& *sym == kw::Fn
241+
&& !is_raw =>
242+
{
243+
false
244+
}
245+
246+
// `>` + `(`: `f::<u8>()`
247+
// `>>` + `(`: `collect::<Vec<_>>()`
248+
(Tok(Token { kind: Gt | BinOp(Shr), .. }, _), Del(_, Parenthesis, _)) => false,
249+
250+
// IDENT + `!`: `println!()`, but `if !x { ... }` needs a space after the `if`
251+
(Tok(Token { kind: Ident(sym, is_raw), span }, _), Tok(Token { kind: Not, .. }, _))
252+
if !Ident::new(*sym, *span).is_reserved() || *is_raw =>
253+
{
254+
false
255+
}
256+
257+
// ANYTHING-BUT-`macro_rules` + `!` + NON-PUNCT-OR-BRACE: `foo!()`, `vec![]`,
258+
// `if !cond { ... }`, but not `macro_rules! m { ... }`
259+
(Tok(Token { kind: Not, .. }, _), tt2) if is_punct(tt2) => true,
260+
(Tok(Token { kind: Not, .. }, _), Del(_, Brace, _)) => true,
261+
(Tok(Token { kind: Not, .. }, _), _) =>
262+
if let Some(prev) = prev
263+
&& let Tok(Token { kind: Ident(sym, is_raw), .. }, _) = prev
264+
&& *sym == sym::macro_rules
265+
&& !is_raw
266+
{
267+
true
268+
} else {
269+
false
270+
}
181271

182-
// IDENT + `(`: `f(3)`
183-
//
184-
// FIXME: Incorrect cases:
185-
// - Let: `let(a, b) = (1, 2)`
186-
(Tok(Token { kind: Ident(..), .. }, _), Del(_, Parenthesis, _)) => false,
272+
// `~` + `const`: `impl ~const Clone`
273+
(Tok(Token { kind: Tilde, .. }, _), Tok(Token { kind: Ident(sym, is_raw), .. }, _))
274+
if *sym == kw::Const && !is_raw =>
275+
{
276+
false
277+
}
187278

188-
// `#` + `[`: `#[attr]`
189-
(Tok(Token { kind: Pound, .. }, _), Del(_, Bracket, _)) => false,
279+
// `?` + `Sized`: `dyn ?Sized`
280+
(Tok(Token { kind: Question, .. }, _), Tok(Token { kind: Ident(sym, is_raw), .. }, _))
281+
if *sym == sym::Sized && !is_raw =>
282+
{
283+
false
284+
}
190285

191286
_ => true,
192287
}
@@ -583,14 +678,19 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere
583678
}
584679

585680
fn print_tts(&mut self, tts: &TokenStream, convert_dollar_crate: bool) {
681+
let mut prev = None;
586682
let mut iter = tts.trees().peekable();
587683
while let Some(tt) = iter.next() {
588684
self.print_tt(tt, convert_dollar_crate);
589685
if let Some(next) = iter.peek() {
590-
if space_between(tt, next) {
686+
if space_between(prev, tt, next) {
591687
self.space();
688+
} else {
689+
// There must be a space between two punctuation tokens.
690+
assert!(!is_punct(tt) || !is_punct(next));
592691
}
593692
}
693+
prev = Some(tt);
594694
}
595695
}
596696

‎compiler/rustc_span/src/symbol.rs

+1
Original file line numberDiff line numberDiff line change
@@ -302,6 +302,7 @@ symbols! {
302302
Saturating,
303303
Send,
304304
SeqCst,
305+
Sized,
305306
SliceIndex,
306307
SliceIter,
307308
Some,

‎tests/pretty/ast-stmt-expr-attr.rs

+3-3
Original file line numberDiff line numberDiff line change
@@ -113,11 +113,11 @@ fn syntax() {
113113
let _ = #[attr] continue;
114114
let _ = #[attr] return;
115115
let _ = #[attr] foo!();
116-
let _ = #[attr] foo!(#! [attr]);
116+
let _ = #[attr] foo!(# ![attr]);
117117
let _ = #[attr] foo![];
118-
let _ = #[attr] foo![#! [attr]];
118+
let _ = #[attr] foo![# ![attr]];
119119
let _ = #[attr] foo! {};
120-
let _ = #[attr] foo! { #! [attr] };
120+
let _ = #[attr] foo! { # ![attr] };
121121
let _ = #[attr] Foo { bar: baz };
122122
let _ = #[attr] Foo { ..foo };
123123
let _ = #[attr] Foo { bar: baz, ..foo };

‎tests/pretty/cast-lt.pp

+1-1
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,6 @@
88
// pretty-mode:expanded
99
// pp-exact:cast-lt.pp
1010

11-
macro_rules! negative { ($e : expr) => { $e < 0 } }
11+
macro_rules! negative { ($e: expr) => { $e < 0 } }
1212

1313
fn main() { (1 as i32) < 0; }

‎tests/pretty/delimited-token-groups.rs

+5-5
Original file line numberDiff line numberDiff line change
@@ -2,23 +2,23 @@
22

33
#![feature(rustc_attrs)]
44

5-
macro_rules! mac { ($($tt : tt) *) => () }
5+
macro_rules! mac { ($($tt: tt) *) => () }
66

77
mac! {
8-
struct S { field1 : u8, field2 : u16, } impl Clone for S
8+
struct S { field1: u8, field2: u16, } impl Clone for S
99
{
1010
fn clone() -> S
1111
{
12-
panic! () ;
12+
panic!();
1313

1414
}
1515
}
1616
}
1717

1818
mac! {
1919
a(aaaaaaaa aaaaaaaa aaaaaaaa aaaaaaaa aaaaaaaa aaaaaaaa aaaaaaaa aaaaaaaa
20-
aaaaaaaa aaaaaaaa) a
21-
[aaaaaaaa aaaaaaaa aaaaaaaa aaaaaaaa aaaaaaaa aaaaaaaa aaaaaaaa aaaaaaaa
20+
aaaaaaaa aaaaaaaa)
21+
a[aaaaaaaa aaaaaaaa aaaaaaaa aaaaaaaa aaaaaaaa aaaaaaaa aaaaaaaa aaaaaaaa
2222
aaaaaaaa aaaaaaaa] a
2323
{
2424
aaaaaaaa aaaaaaaa aaaaaaaa aaaaaaaa aaaaaaaa aaaaaaaa aaaaaaaa

‎tests/pretty/macro.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,6 @@
22

33
#![feature(decl_macro)]
44

5-
pub(crate) macro mac { ($arg : expr) => { $arg + $arg } }
5+
pub(crate) macro mac { ($arg: expr) => { $arg + $arg } }
66

77
fn main() {}

‎tests/pretty/macro_rules.rs

+7-7
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,19 @@
11
// pp-exact
22

3-
macro_rules! brace { () => {} ; }
3+
macro_rules! brace { () => {}; }
44

5-
macro_rules! bracket[() => {} ;];
5+
macro_rules! bracket[() => {};];
66

7-
macro_rules! paren(() => {} ;);
7+
macro_rules! paren(() => {};);
88

99
macro_rules! matcher_brackets {
10-
(paren) => {} ; (bracket) => {} ; (brace) => {} ;
10+
(paren) => {}; (bracket) => {}; (brace) => {};
1111
}
1212

1313
macro_rules! all_fragments {
14-
($b : block, $e : expr, $i : ident, $it : item, $l : lifetime, $lit :
15-
literal, $m : meta, $p : pat, $pth : path, $s : stmt, $tt : tt, $ty : ty,
16-
$vis : vis) => {} ;
14+
($b: block, $e: expr, $i: ident, $it: item, $l: lifetime, $lit: literal,
15+
$m: meta, $p: pat, $pth: path, $s: stmt, $tt: tt, $ty: ty, $vis: vis) =>
16+
{};
1717
}
1818

1919
fn main() {}

‎tests/pretty/offset_of.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
11
// pp-exact
22
#![feature(offset_of)]
33

4-
fn main() { std::mem::offset_of!(std :: ops :: Range < usize >, end); }
4+
fn main() { std::mem::offset_of!(std::ops::Range < usize > , end); }

‎tests/pretty/stmt_expr_attributes.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -113,7 +113,7 @@ fn _8() {
113113
}
114114

115115
fn _9() {
116-
macro_rules! stmt_mac { () => { let _ = () ; } }
116+
macro_rules! stmt_mac { () => { let _ = (); } }
117117

118118
#[rustc_dummy]
119119
stmt_mac!();

‎tests/run-make/rustc-macro-dep-files/foo.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,6 @@ use proc_macro::TokenStream;
77
#[proc_macro_derive(A)]
88
pub fn derive(input: TokenStream) -> TokenStream {
99
let input = input.to_string();
10-
assert!(input.contains("struct A ;"));
10+
assert!(input.contains("struct A;"));
1111
"struct B;".parse().unwrap()
1212
}

‎tests/ui/async-await/issues/issue-60674.stdout

+3-3
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
1-
async fn f(mut x : u8) {}
2-
async fn g((mut x, y, mut z) : (u8, u8, u8)) {}
3-
async fn g(mut x : u8, (a, mut b, c) : (u8, u8, u8), y : u8) {}
1+
async fn f(mut x: u8) {}
2+
async fn g((mut x, y, mut z): (u8, u8, u8)) {}
3+
async fn g(mut x: u8, (a, mut b, c): (u8, u8, u8), y: u8) {}

‎tests/ui/hygiene/unpretty-debug.stdout

+1-1
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
#![feature /* 0#0 */(no_core)]
99
#![no_core /* 0#0 */]
1010

11-
macro_rules! foo /* 0#0 */ { ($x : ident) => { y + $x } }
11+
macro_rules! foo /* 0#0 */ { ($x: ident) => { y + $x } }
1212

1313
fn bar /* 0#0 */() {
1414
let x /* 0#0 */ = 1;

‎tests/ui/infinite/issue-41731-infinite-macro-print.stderr

+6-6
Original file line numberDiff line numberDiff line change
@@ -14,13 +14,13 @@ LL | stack!("overflow");
1414
| ^^^^^^^^^^^^^^^^^^
1515
|
1616
= note: expanding `stack! { "overflow" }`
17-
= note: to `print! (stack! ("overflow")) ;`
18-
= note: expanding `print! { stack! ("overflow") }`
19-
= note: to `{ $crate :: io :: _print($crate :: format_args! (stack! ("overflow"))) ; }`
17+
= note: to `print!(stack!("overflow"));`
18+
= note: expanding `print! { stack!("overflow") }`
19+
= note: to `{ $crate::io::_print($crate::format_args!(stack!("overflow"))); }`
2020
= note: expanding `stack! { "overflow" }`
21-
= note: to `print! (stack! ("overflow")) ;`
22-
= note: expanding `print! { stack! ("overflow") }`
23-
= note: to `{ $crate :: io :: _print($crate :: format_args! (stack! ("overflow"))) ; }`
21+
= note: to `print!(stack!("overflow"));`
22+
= note: expanding `print! { stack!("overflow") }`
23+
= note: to `{ $crate::io::_print($crate::format_args!(stack!("overflow"))); }`
2424

2525
error: format argument must be a string literal
2626
--> $DIR/issue-41731-infinite-macro-print.rs:14:5

‎tests/ui/infinite/issue-41731-infinite-macro-println.stderr

+6-6
Original file line numberDiff line numberDiff line change
@@ -14,13 +14,13 @@ LL | stack!("overflow");
1414
| ^^^^^^^^^^^^^^^^^^
1515
|
1616
= note: expanding `stack! { "overflow" }`
17-
= note: to `println! (stack! ("overflow")) ;`
18-
= note: expanding `println! { stack! ("overflow") }`
19-
= note: to `{ $crate :: io :: _print($crate :: format_args_nl! (stack! ("overflow"))) ; }`
17+
= note: to `println!(stack!("overflow"));`
18+
= note: expanding `println! { stack!("overflow") }`
19+
= note: to `{ $crate::io::_print($crate::format_args_nl!(stack!("overflow"))); }`
2020
= note: expanding `stack! { "overflow" }`
21-
= note: to `println! (stack! ("overflow")) ;`
22-
= note: expanding `println! { stack! ("overflow") }`
23-
= note: to `{ $crate :: io :: _print($crate :: format_args_nl! (stack! ("overflow"))) ; }`
21+
= note: to `println!(stack!("overflow"));`
22+
= note: expanding `println! { stack!("overflow") }`
23+
= note: to `{ $crate::io::_print($crate::format_args_nl!(stack!("overflow"))); }`
2424

2525
error: format argument must be a string literal
2626
--> $DIR/issue-41731-infinite-macro-println.rs:14:5

0 commit comments

Comments
 (0)