Skip to content

Commit a9cb8ee

Browse files
borsbors
committed
Auto merge of #114571 - nnethercote:improve-print_tts, r=petrochenkov
Improve `print_tts` By slightly changing the meaning of `tokenstream::Spacing` we can greatly improve the output of `print_tts`. r? `@ghost`
2 parents d86d65b + 940c885 commit a9cb8ee

File tree

78 files changed

+816
-525
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

78 files changed

+816
-525
lines changed

‎compiler/rustc_ast/src/attr/mod.rs

+3-3
Original file line numberDiff line numberDiff line change
@@ -387,7 +387,7 @@ impl MetaItemKind {
387387
tokens: &mut impl Iterator<Item = &'a TokenTree>,
388388
) -> Option<MetaItemKind> {
389389
match tokens.next() {
390-
Some(TokenTree::Delimited(_, Delimiter::Invisible, inner_tokens)) => {
390+
Some(TokenTree::Delimited(.., Delimiter::Invisible, inner_tokens)) => {
391391
MetaItemKind::name_value_from_tokens(&mut inner_tokens.trees())
392392
}
393393
Some(TokenTree::Token(token, _)) => {
@@ -401,7 +401,7 @@ impl MetaItemKind {
401401
tokens: &mut iter::Peekable<impl Iterator<Item = &'a TokenTree>>,
402402
) -> Option<MetaItemKind> {
403403
match tokens.peek() {
404-
Some(TokenTree::Delimited(_, Delimiter::Parenthesis, inner_tokens)) => {
404+
Some(TokenTree::Delimited(.., Delimiter::Parenthesis, inner_tokens)) => {
405405
let inner_tokens = inner_tokens.clone();
406406
tokens.next();
407407
MetaItemKind::list_from_tokens(inner_tokens).map(MetaItemKind::List)
@@ -524,7 +524,7 @@ impl NestedMetaItem {
524524
tokens.next();
525525
return Some(NestedMetaItem::Lit(lit));
526526
}
527-
Some(TokenTree::Delimited(_, Delimiter::Invisible, inner_tokens)) => {
527+
Some(TokenTree::Delimited(.., Delimiter::Invisible, inner_tokens)) => {
528528
tokens.next();
529529
return NestedMetaItem::from_tokens(&mut inner_tokens.trees().peekable());
530530
}

‎compiler/rustc_ast/src/mut_visit.rs

+2-2
Original file line numberDiff line numberDiff line change
@@ -682,7 +682,7 @@ pub fn visit_attr_tt<T: MutVisitor>(tt: &mut AttrTokenTree, vis: &mut T) {
682682
AttrTokenTree::Token(token, _) => {
683683
visit_token(token, vis);
684684
}
685-
AttrTokenTree::Delimited(DelimSpan { open, close }, _delim, tts) => {
685+
AttrTokenTree::Delimited(DelimSpan { open, close }, _spacing, _delim, tts) => {
686686
vis.visit_span(open);
687687
vis.visit_span(close);
688688
visit_attr_tts(tts, vis);
@@ -709,7 +709,7 @@ pub fn visit_tt<T: MutVisitor>(tt: &mut TokenTree, vis: &mut T) {
709709
TokenTree::Token(token, _) => {
710710
visit_token(token, vis);
711711
}
712-
TokenTree::Delimited(DelimSpan { open, close }, _delim, tts) => {
712+
TokenTree::Delimited(DelimSpan { open, close }, _spacing, _delim, tts) => {
713713
vis.visit_span(open);
714714
vis.visit_span(close);
715715
visit_tts(tts, vis);

‎compiler/rustc_ast/src/tokenstream.rs

+99-38
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@ pub enum TokenTree {
4646
/// delimiters are implicitly represented by `Delimited`.
4747
Token(Token, Spacing),
4848
/// A delimited sequence of token trees.
49-
Delimited(DelimSpan, Delimiter, TokenStream),
49+
Delimited(DelimSpan, DelimSpacing, Delimiter, TokenStream),
5050
}
5151

5252
// Ensure all fields of `TokenTree` are `DynSend` and `DynSync`.
@@ -62,11 +62,11 @@ where
6262
}
6363

6464
impl TokenTree {
65-
/// Checks if this `TokenTree` is equal to the other, regardless of span information.
65+
/// Checks if this `TokenTree` is equal to the other, regardless of span/spacing information.
6666
pub fn eq_unspanned(&self, other: &TokenTree) -> bool {
6767
match (self, other) {
6868
(TokenTree::Token(token, _), TokenTree::Token(token2, _)) => token.kind == token2.kind,
69-
(TokenTree::Delimited(_, delim, tts), TokenTree::Delimited(_, delim2, tts2)) => {
69+
(TokenTree::Delimited(.., delim, tts), TokenTree::Delimited(.., delim2, tts2)) => {
7070
delim == delim2 && tts.eq_unspanned(tts2)
7171
}
7272
_ => false,
@@ -99,6 +99,11 @@ impl TokenTree {
9999
TokenTree::Token(Token::new(kind, span), Spacing::Joint)
100100
}
101101

102+
/// Create a `TokenTree::Token` with joint-hidden spacing.
103+
pub fn token_joint_hidden(kind: TokenKind, span: Span) -> TokenTree {
104+
TokenTree::Token(Token::new(kind, span), Spacing::JointHidden)
105+
}
106+
102107
pub fn uninterpolate(&self) -> Cow<'_, TokenTree> {
103108
match self {
104109
TokenTree::Token(token, spacing) => match token.uninterpolate() {
@@ -183,7 +188,7 @@ pub struct AttrTokenStream(pub Lrc<Vec<AttrTokenTree>>);
183188
#[derive(Clone, Debug, Encodable, Decodable)]
184189
pub enum AttrTokenTree {
185190
Token(Token, Spacing),
186-
Delimited(DelimSpan, Delimiter, AttrTokenStream),
191+
Delimited(DelimSpan, DelimSpacing, Delimiter, AttrTokenStream),
187192
/// Stores the attributes for an attribute target,
188193
/// along with the tokens for that attribute target.
189194
/// See `AttributesData` for more information
@@ -208,9 +213,14 @@ impl AttrTokenStream {
208213
AttrTokenTree::Token(inner, spacing) => {
209214
smallvec![TokenTree::Token(inner.clone(), *spacing)].into_iter()
210215
}
211-
AttrTokenTree::Delimited(span, delim, stream) => {
212-
smallvec![TokenTree::Delimited(*span, *delim, stream.to_tokenstream()),]
213-
.into_iter()
216+
AttrTokenTree::Delimited(span, spacing, delim, stream) => {
217+
smallvec![TokenTree::Delimited(
218+
*span,
219+
*spacing,
220+
*delim,
221+
stream.to_tokenstream()
222+
),]
223+
.into_iter()
214224
}
215225
AttrTokenTree::Attributes(data) => {
216226
let idx = data
@@ -230,7 +240,7 @@ impl AttrTokenStream {
230240
let mut found = false;
231241
// Check the last two trees (to account for a trailing semi)
232242
for tree in target_tokens.iter_mut().rev().take(2) {
233-
if let TokenTree::Delimited(span, delim, delim_tokens) = tree {
243+
if let TokenTree::Delimited(span, spacing, delim, delim_tokens) = tree {
234244
// Inner attributes are only supported on extern blocks, functions,
235245
// impls, and modules. All of these have their inner attributes
236246
// placed at the beginning of the rightmost outermost braced group:
@@ -250,7 +260,7 @@ impl AttrTokenStream {
250260
stream.push_stream(inner_attr.tokens());
251261
}
252262
stream.push_stream(delim_tokens.clone());
253-
*tree = TokenTree::Delimited(*span, *delim, stream);
263+
*tree = TokenTree::Delimited(*span, *spacing, *delim, stream);
254264
found = true;
255265
break;
256266
}
@@ -303,21 +313,64 @@ pub struct AttributesData {
303313
#[derive(Clone, Debug, Default, Encodable, Decodable)]
304314
pub struct TokenStream(pub(crate) Lrc<Vec<TokenTree>>);
305315

306-
/// Similar to `proc_macro::Spacing`, but for tokens.
307-
///
308-
/// Note that all `ast::TokenTree::Token` instances have a `Spacing`, but when
309-
/// we convert to `proc_macro::TokenTree` for proc macros only `Punct`
310-
/// `TokenTree`s have a `proc_macro::Spacing`.
316+
/// Indicates whether a token can join with the following token to form a
317+
/// compound token. Used for conversions to `proc_macro::Spacing`. Also used to
318+
/// guide pretty-printing, which is where the `JointHidden` value (which isn't
319+
/// part of `proc_macro::Spacing`) comes in useful.
311320
#[derive(Clone, Copy, Debug, PartialEq, Encodable, Decodable, HashStable_Generic)]
312321
pub enum Spacing {
313-
/// The token is not immediately followed by an operator token (as
314-
/// determined by `Token::is_op`). E.g. a `+` token is `Alone` in `+ =`,
315-
/// `+/*foo*/=`, `+ident`, and `+()`.
322+
/// The token cannot join with the following token to form a compound
323+
/// token.
324+
///
325+
/// In token streams parsed from source code, the compiler will use `Alone`
326+
/// for any token immediately followed by whitespace, a non-doc comment, or
327+
/// EOF.
328+
///
329+
/// When constructing token streams within the compiler, use this for each
330+
/// token that (a) should be pretty-printed with a space after it, or (b)
331+
/// is the last token in the stream. (In the latter case the choice of
332+
/// spacing doesn't matter because it is never used for the last token. We
333+
/// arbitrarily use `Alone`.)
334+
///
335+
/// Converts to `proc_macro::Spacing::Alone`, and
336+
/// `proc_macro::Spacing::Alone` converts back to this.
316337
Alone,
317338

318-
/// The token is immediately followed by an operator token. E.g. a `+`
319-
/// token is `Joint` in `+=` and `++`.
339+
/// The token can join with the following token to form a compound token.
340+
///
341+
/// In token streams parsed from source code, the compiler will use `Joint`
342+
/// for any token immediately followed by punctuation (as determined by
343+
/// `Token::is_punct`).
344+
///
345+
/// When constructing token streams within the compiler, use this for each
346+
/// token that (a) should be pretty-printed without a space after it, and
347+
/// (b) is followed by a punctuation token.
348+
///
349+
/// Converts to `proc_macro::Spacing::Joint`, and
350+
/// `proc_macro::Spacing::Joint` converts back to this.
320351
Joint,
352+
353+
/// The token can join with the following token to form a compound token,
354+
/// but this will not be visible at the proc macro level. (This is what the
355+
/// `Hidden` means; see below.)
356+
///
357+
/// In token streams parsed from source code, the compiler will use
358+
/// `JointHidden` for any token immediately followed by anything not
359+
/// covered by the `Alone` and `Joint` cases: an identifier, lifetime,
360+
/// literal, delimiter, doc comment.
361+
///
362+
/// When constructing token streams, use this for each token that (a)
363+
/// should be pretty-printed without a space after it, and (b) is followed
364+
/// by a non-punctuation token.
365+
///
366+
/// Converts to `proc_macro::Spacing::Alone`, but
367+
/// `proc_macro::Spacing::Alone` converts back to `token::Spacing::Alone`.
368+
/// Because of that, pretty-printing of `TokenStream`s produced by proc
369+
/// macros is unavoidably uglier (with more whitespace between tokens) than
370+
/// pretty-printing of `TokenStream`'s produced by other means (i.e. parsed
371+
/// source code, internally constructed token streams, and token streams
372+
/// produced by declarative macros).
373+
JointHidden,
321374
}
322375

323376
impl TokenStream {
@@ -421,21 +474,14 @@ impl TokenStream {
421474
self
422475
}
423476

424-
/// Create a token stream containing a single token with alone spacing.
477+
/// Create a token stream containing a single token with alone spacing. The
478+
/// spacing used for the final token in a constructed stream doesn't matter
479+
/// because it's never used. In practice we arbitrarily use
480+
/// `Spacing::Alone`.
425481
pub fn token_alone(kind: TokenKind, span: Span) -> TokenStream {
426482
TokenStream::new(vec![TokenTree::token_alone(kind, span)])
427483
}
428484

429-
/// Create a token stream containing a single token with joint spacing.
430-
pub fn token_joint(kind: TokenKind, span: Span) -> TokenStream {
431-
TokenStream::new(vec![TokenTree::token_joint(kind, span)])
432-
}
433-
434-
/// Create a token stream containing a single `Delimited`.
435-
pub fn delimited(span: DelimSpan, delim: Delimiter, tts: TokenStream) -> TokenStream {
436-
TokenStream::new(vec![TokenTree::Delimited(span, delim, tts)])
437-
}
438-
439485
pub fn from_ast(node: &(impl HasAttrs + HasSpan + HasTokens + fmt::Debug)) -> TokenStream {
440486
let Some(tokens) = node.tokens() else {
441487
panic!("missing tokens for node at {:?}: {:?}", node.span(), node);
@@ -482,6 +528,7 @@ impl TokenStream {
482528
}
483529
token::Interpolated(nt) => TokenTree::Delimited(
484530
DelimSpan::from_single(token.span),
531+
DelimSpacing::new(Spacing::JointHidden, spacing),
485532
Delimiter::Invisible,
486533
TokenStream::from_nonterminal_ast(&nt.0).flattened(),
487534
),
@@ -492,8 +539,8 @@ impl TokenStream {
492539
fn flatten_token_tree(tree: &TokenTree) -> TokenTree {
493540
match tree {
494541
TokenTree::Token(token, spacing) => TokenStream::flatten_token(token, *spacing),
495-
TokenTree::Delimited(span, delim, tts) => {
496-
TokenTree::Delimited(*span, *delim, tts.flattened())
542+
TokenTree::Delimited(span, spacing, delim, tts) => {
543+
TokenTree::Delimited(*span, *spacing, *delim, tts.flattened())
497544
}
498545
}
499546
}
@@ -503,7 +550,7 @@ impl TokenStream {
503550
fn can_skip(stream: &TokenStream) -> bool {
504551
stream.trees().all(|tree| match tree {
505552
TokenTree::Token(token, _) => !matches!(token.kind, token::Interpolated(_)),
506-
TokenTree::Delimited(_, _, inner) => can_skip(inner),
553+
TokenTree::Delimited(.., inner) => can_skip(inner),
507554
})
508555
}
509556

@@ -517,7 +564,7 @@ impl TokenStream {
517564
// If `vec` is not empty, try to glue `tt` onto its last token. The return
518565
// value indicates if gluing took place.
519566
fn try_glue_to_last(vec: &mut Vec<TokenTree>, tt: &TokenTree) -> bool {
520-
if let Some(TokenTree::Token(last_tok, Spacing::Joint)) = vec.last()
567+
if let Some(TokenTree::Token(last_tok, Spacing::Joint | Spacing::JointHidden)) = vec.last()
521568
&& let TokenTree::Token(tok, spacing) = tt
522569
&& let Some(glued_tok) = last_tok.glue(tok)
523570
{
@@ -592,9 +639,10 @@ impl TokenStream {
592639

593640
&TokenTree::Token(..) => i += 1,
594641

595-
&TokenTree::Delimited(sp, delim, ref delim_stream) => {
642+
&TokenTree::Delimited(sp, spacing, delim, ref delim_stream) => {
596643
if let Some(desugared_delim_stream) = desugar_inner(delim_stream.clone()) {
597-
let new_tt = TokenTree::Delimited(sp, delim, desugared_delim_stream);
644+
let new_tt =
645+
TokenTree::Delimited(sp, spacing, delim, desugared_delim_stream);
598646
Lrc::make_mut(&mut stream.0)[i] = new_tt;
599647
modified = true;
600648
}
@@ -622,10 +670,11 @@ impl TokenStream {
622670
num_of_hashes = cmp::max(num_of_hashes, count);
623671
}
624672

625-
// `/// foo` becomes `doc = r"foo"`.
673+
// `/// foo` becomes `[doc = r"foo"]`.
626674
let delim_span = DelimSpan::from_single(span);
627675
let body = TokenTree::Delimited(
628676
delim_span,
677+
DelimSpacing::new(Spacing::JointHidden, Spacing::Alone),
629678
Delimiter::Bracket,
630679
[
631680
TokenTree::token_alone(token::Ident(sym::doc, false), span),
@@ -641,7 +690,7 @@ impl TokenStream {
641690

642691
if attr_style == AttrStyle::Inner {
643692
vec![
644-
TokenTree::token_alone(token::Pound, span),
693+
TokenTree::token_joint(token::Pound, span),
645694
TokenTree::token_alone(token::Not, span),
646695
body,
647696
]
@@ -738,6 +787,18 @@ impl DelimSpan {
738787
}
739788
}
740789

790+
#[derive(Copy, Clone, Debug, PartialEq, Encodable, Decodable, HashStable_Generic)]
791+
pub struct DelimSpacing {
792+
pub open: Spacing,
793+
pub close: Spacing,
794+
}
795+
796+
impl DelimSpacing {
797+
pub fn new(open: Spacing, close: Spacing) -> DelimSpacing {
798+
DelimSpacing { open, close }
799+
}
800+
}
801+
741802
// Some types are used a lot. Make sure they don't unintentionally get bigger.
742803
#[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
743804
mod size_asserts {

0 commit comments

Comments
 (0)