Skip to content

Commit 7aef878

Browse files
borsbors
committed
Auto merge of #127558 - nnethercote:more-Attribute-cleanups, r=petrochenkov
More attribute cleanups A follow-up to #127308. r? `@petrochenkov`
2 parents c1e3f03 + 478ba59 commit 7aef878

File tree

6 files changed

+150
-158
lines changed

6 files changed

+150
-158
lines changed

‎compiler/rustc_ast/src/attr/mod.rs

+10-13
Original file line numberDiff line numberDiff line change
@@ -202,21 +202,18 @@ impl Attribute {
202202
}
203203
}
204204

205-
// Named `get_tokens` to distinguish it from the `<Attribute as HasTokens>::tokens` method.
206-
pub fn get_tokens(&self) -> TokenStream {
207-
match &self.kind {
208-
AttrKind::Normal(normal) => TokenStream::new(
209-
normal
210-
.tokens
211-
.as_ref()
212-
.unwrap_or_else(|| panic!("attribute is missing tokens: {self:?}"))
213-
.to_attr_token_stream()
214-
.to_token_trees(),
215-
),
216-
&AttrKind::DocComment(comment_kind, data) => TokenStream::token_alone(
205+
pub fn token_trees(&self) -> Vec<TokenTree> {
206+
match self.kind {
207+
AttrKind::Normal(ref normal) => normal
208+
.tokens
209+
.as_ref()
210+
.unwrap_or_else(|| panic!("attribute is missing tokens: {self:?}"))
211+
.to_attr_token_stream()
212+
.to_token_trees(),
213+
AttrKind::DocComment(comment_kind, data) => vec![TokenTree::token_alone(
217214
token::DocComment(comment_kind, self.style, data),
218215
self.span,
219-
),
216+
)],
220217
}
221218
}
222219
}

‎compiler/rustc_ast/src/tokenstream.rs

+72-64
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@
1616
use crate::ast::{AttrStyle, StmtKind};
1717
use crate::ast_traits::{HasAttrs, HasTokens};
1818
use crate::token::{self, Delimiter, Nonterminal, Token, TokenKind};
19-
use crate::AttrVec;
19+
use crate::{AttrVec, Attribute};
2020

2121
use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
2222
use rustc_data_structures::sync::{self, Lrc};
@@ -179,11 +179,10 @@ impl AttrTokenStream {
179179
AttrTokenStream(Lrc::new(tokens))
180180
}
181181

182-
/// Converts this `AttrTokenStream` to a plain `Vec<TokenTree>`.
183-
/// During conversion, `AttrTokenTree::AttrsTarget` get 'flattened'
184-
/// back to a `TokenStream` of the form `outer_attr attr_target`.
185-
/// If there are inner attributes, they are inserted into the proper
186-
/// place in the attribute target tokens.
182+
/// Converts this `AttrTokenStream` to a plain `Vec<TokenTree>`. During
183+
/// conversion, any `AttrTokenTree::AttrsTarget` gets "flattened" back to a
184+
/// `TokenStream`, as described in the comment on
185+
/// `attrs_and_tokens_to_token_trees`.
187186
pub fn to_token_trees(&self) -> Vec<TokenTree> {
188187
let mut res = Vec::with_capacity(self.0.len());
189188
for tree in self.0.iter() {
@@ -200,67 +199,84 @@ impl AttrTokenStream {
200199
))
201200
}
202201
AttrTokenTree::AttrsTarget(target) => {
203-
let idx = target
204-
.attrs
205-
.partition_point(|attr| matches!(attr.style, crate::AttrStyle::Outer));
206-
let (outer_attrs, inner_attrs) = target.attrs.split_at(idx);
207-
208-
let mut target_tokens = target.tokens.to_attr_token_stream().to_token_trees();
209-
if !inner_attrs.is_empty() {
210-
let mut found = false;
211-
// Check the last two trees (to account for a trailing semi)
212-
for tree in target_tokens.iter_mut().rev().take(2) {
213-
if let TokenTree::Delimited(span, spacing, delim, delim_tokens) = tree {
214-
// Inner attributes are only supported on extern blocks, functions,
215-
// impls, and modules. All of these have their inner attributes
216-
// placed at the beginning of the rightmost outermost braced group:
217-
// e.g. fn foo() { #![my_attr] }
218-
//
219-
// Therefore, we can insert them back into the right location
220-
// without needing to do any extra position tracking.
221-
//
222-
// Note: Outline modules are an exception - they can
223-
// have attributes like `#![my_attr]` at the start of a file.
224-
// Support for custom attributes in this position is not
225-
// properly implemented - we always synthesize fake tokens,
226-
// so we never reach this code.
227-
228-
let mut stream = TokenStream::default();
229-
for inner_attr in inner_attrs {
230-
stream.push_stream(inner_attr.get_tokens());
231-
}
232-
stream.push_stream(delim_tokens.clone());
233-
*tree = TokenTree::Delimited(*span, *spacing, *delim, stream);
234-
found = true;
235-
break;
236-
}
237-
}
238-
239-
assert!(
240-
found,
241-
"Failed to find trailing delimited group in: {target_tokens:?}"
242-
);
243-
}
244-
for attr in outer_attrs {
245-
res.extend(attr.get_tokens().0.iter().cloned());
246-
}
247-
res.extend(target_tokens);
202+
attrs_and_tokens_to_token_trees(&target.attrs, &target.tokens, &mut res);
248203
}
249204
}
250205
}
251206
res
252207
}
253208
}
254209

210+
// Converts multiple attributes and the tokens for a target AST node into token trees, and appends
211+
// them to `res`.
212+
//
213+
// Example: if the AST node is "fn f() { blah(); }", then:
214+
// - Simple if no attributes are present, e.g. "fn f() { blah(); }"
215+
// - Simple if only outer attribute are present, e.g. "#[outer1] #[outer2] fn f() { blah(); }"
216+
// - Trickier if inner attributes are present, because they must be moved within the AST node's
217+
// tokens, e.g. "#[outer] fn f() { #![inner] blah() }"
218+
fn attrs_and_tokens_to_token_trees(
219+
attrs: &[Attribute],
220+
target_tokens: &LazyAttrTokenStream,
221+
res: &mut Vec<TokenTree>,
222+
) {
223+
let idx = attrs.partition_point(|attr| matches!(attr.style, crate::AttrStyle::Outer));
224+
let (outer_attrs, inner_attrs) = attrs.split_at(idx);
225+
226+
// Add outer attribute tokens.
227+
for attr in outer_attrs {
228+
res.extend(attr.token_trees());
229+
}
230+
231+
// Add target AST node tokens.
232+
res.extend(target_tokens.to_attr_token_stream().to_token_trees());
233+
234+
// Insert inner attribute tokens.
235+
if !inner_attrs.is_empty() {
236+
let mut found = false;
237+
// Check the last two trees (to account for a trailing semi)
238+
for tree in res.iter_mut().rev().take(2) {
239+
if let TokenTree::Delimited(span, spacing, delim, delim_tokens) = tree {
240+
// Inner attributes are only supported on extern blocks, functions,
241+
// impls, and modules. All of these have their inner attributes
242+
// placed at the beginning of the rightmost outermost braced group:
243+
// e.g. fn foo() { #![my_attr] }
244+
//
245+
// Therefore, we can insert them back into the right location
246+
// without needing to do any extra position tracking.
247+
//
248+
// Note: Outline modules are an exception - they can
249+
// have attributes like `#![my_attr]` at the start of a file.
250+
// Support for custom attributes in this position is not
251+
// properly implemented - we always synthesize fake tokens,
252+
// so we never reach this code.
253+
let mut tts = vec![];
254+
for inner_attr in inner_attrs {
255+
tts.extend(inner_attr.token_trees());
256+
}
257+
tts.extend(delim_tokens.0.iter().cloned());
258+
let stream = TokenStream::new(tts);
259+
*tree = TokenTree::Delimited(*span, *spacing, *delim, stream);
260+
found = true;
261+
break;
262+
}
263+
}
264+
assert!(found, "Failed to find trailing delimited group in: {res:?}");
265+
}
266+
}
267+
255268
/// Stores the tokens for an attribute target, along
256269
/// with its attributes.
257270
///
258271
/// This is constructed during parsing when we need to capture
259-
/// tokens.
272+
/// tokens, for `cfg` and `cfg_attr` attributes.
260273
///
261274
/// For example, `#[cfg(FALSE)] struct Foo {}` would
262275
/// have an `attrs` field containing the `#[cfg(FALSE)]` attr,
263276
/// and a `tokens` field storing the (unparsed) tokens `struct Foo {}`
277+
///
278+
/// The `cfg`/`cfg_attr` processing occurs in
279+
/// `StripUnconfigured::configure_tokens`.
264280
#[derive(Clone, Debug, Encodable, Decodable)]
265281
pub struct AttrsTarget {
266282
/// Attributes, both outer and inner.
@@ -437,18 +453,10 @@ impl TokenStream {
437453
}
438454

439455
pub fn from_ast(node: &(impl HasAttrs + HasTokens + fmt::Debug)) -> TokenStream {
440-
let Some(tokens) = node.tokens() else {
441-
panic!("missing tokens for node: {:?}", node);
442-
};
443-
let attrs = node.attrs();
444-
let attr_stream = if attrs.is_empty() {
445-
tokens.to_attr_token_stream()
446-
} else {
447-
let target =
448-
AttrsTarget { attrs: attrs.iter().cloned().collect(), tokens: tokens.clone() };
449-
AttrTokenStream::new(vec![AttrTokenTree::AttrsTarget(target)])
450-
};
451-
TokenStream::new(attr_stream.to_token_trees())
456+
let tokens = node.tokens().unwrap_or_else(|| panic!("missing tokens for node: {:?}", node));
457+
let mut tts = vec![];
458+
attrs_and_tokens_to_token_trees(node.attrs(), tokens, &mut tts);
459+
TokenStream::new(tts)
452460
}
453461

454462
pub fn from_nonterminal_ast(nt: &Nonterminal) -> TokenStream {

‎compiler/rustc_expand/src/config.rs

+17-13
Original file line numberDiff line numberDiff line change
@@ -187,6 +187,7 @@ impl<'a> StripUnconfigured<'a> {
187187
.iter()
188188
.filter_map(|tree| match tree.clone() {
189189
AttrTokenTree::AttrsTarget(mut target) => {
190+
// Expand any `cfg_attr` attributes.
190191
target.attrs.flat_map_in_place(|attr| self.process_cfg_attr(&attr));
191192

192193
if self.in_cfg(&target.attrs) {
@@ -195,6 +196,8 @@ impl<'a> StripUnconfigured<'a> {
195196
);
196197
Some(AttrTokenTree::AttrsTarget(target))
197198
} else {
199+
// Remove the target if there's a `cfg` attribute and
200+
// the condition isn't satisfied.
198201
None
199202
}
200203
}
@@ -253,9 +256,9 @@ impl<'a> StripUnconfigured<'a> {
253256
/// Gives a compiler warning when the `cfg_attr` contains no attributes and
254257
/// is in the original source file. Gives a compiler error if the syntax of
255258
/// the attribute is incorrect.
256-
pub(crate) fn expand_cfg_attr(&self, attr: &Attribute, recursive: bool) -> Vec<Attribute> {
259+
pub(crate) fn expand_cfg_attr(&self, cfg_attr: &Attribute, recursive: bool) -> Vec<Attribute> {
257260
let Some((cfg_predicate, expanded_attrs)) =
258-
rustc_parse::parse_cfg_attr(attr, &self.sess.psess)
261+
rustc_parse::parse_cfg_attr(cfg_attr, &self.sess.psess)
259262
else {
260263
return vec![];
261264
};
@@ -264,7 +267,7 @@ impl<'a> StripUnconfigured<'a> {
264267
if expanded_attrs.is_empty() {
265268
self.sess.psess.buffer_lint(
266269
rustc_lint_defs::builtin::UNUSED_ATTRIBUTES,
267-
attr.span,
270+
cfg_attr.span,
268271
ast::CRATE_NODE_ID,
269272
BuiltinLintDiag::CfgAttrNoAttributes,
270273
);
@@ -280,20 +283,21 @@ impl<'a> StripUnconfigured<'a> {
280283
// `#[cfg_attr(false, cfg_attr(true, some_attr))]`.
281284
expanded_attrs
282285
.into_iter()
283-
.flat_map(|item| self.process_cfg_attr(&self.expand_cfg_attr_item(attr, item)))
286+
.flat_map(|item| self.process_cfg_attr(&self.expand_cfg_attr_item(cfg_attr, item)))
284287
.collect()
285288
} else {
286-
expanded_attrs.into_iter().map(|item| self.expand_cfg_attr_item(attr, item)).collect()
289+
expanded_attrs
290+
.into_iter()
291+
.map(|item| self.expand_cfg_attr_item(cfg_attr, item))
292+
.collect()
287293
}
288294
}
289295

290296
fn expand_cfg_attr_item(
291297
&self,
292-
attr: &Attribute,
298+
cfg_attr: &Attribute,
293299
(item, item_span): (ast::AttrItem, Span),
294300
) -> Attribute {
295-
let orig_tokens = attr.get_tokens();
296-
297301
// We are taking an attribute of the form `#[cfg_attr(pred, attr)]`
298302
// and producing an attribute of the form `#[attr]`. We
299303
// have captured tokens for `attr` itself, but we need to
@@ -302,11 +306,11 @@ impl<'a> StripUnconfigured<'a> {
302306

303307
// Use the `#` in `#[cfg_attr(pred, attr)]` as the `#` token
304308
// for `attr` when we expand it to `#[attr]`
305-
let mut orig_trees = orig_tokens.trees();
309+
let mut orig_trees = cfg_attr.token_trees().into_iter();
306310
let TokenTree::Token(pound_token @ Token { kind: TokenKind::Pound, .. }, _) =
307311
orig_trees.next().unwrap().clone()
308312
else {
309-
panic!("Bad tokens for attribute {attr:?}");
313+
panic!("Bad tokens for attribute {cfg_attr:?}");
310314
};
311315

312316
// We don't really have a good span to use for the synthesized `[]`
@@ -320,12 +324,12 @@ impl<'a> StripUnconfigured<'a> {
320324
.unwrap_or_else(|| panic!("Missing tokens for {item:?}"))
321325
.to_attr_token_stream(),
322326
);
323-
let trees = if attr.style == AttrStyle::Inner {
327+
let trees = if cfg_attr.style == AttrStyle::Inner {
324328
// For inner attributes, we do the same thing for the `!` in `#![some_attr]`
325329
let TokenTree::Token(bang_token @ Token { kind: TokenKind::Not, .. }, _) =
326330
orig_trees.next().unwrap().clone()
327331
else {
328-
panic!("Bad tokens for attribute {attr:?}");
332+
panic!("Bad tokens for attribute {cfg_attr:?}");
329333
};
330334
vec![
331335
AttrTokenTree::Token(pound_token, Spacing::Joint),
@@ -340,7 +344,7 @@ impl<'a> StripUnconfigured<'a> {
340344
&self.sess.psess.attr_id_generator,
341345
item,
342346
tokens,
343-
attr.style,
347+
cfg_attr.style,
344348
item_span,
345349
);
346350
if attr.has_name(sym::crate_type) {

‎compiler/rustc_parse/src/lib.rs

+3-3
Original file line numberDiff line numberDiff line change
@@ -157,14 +157,14 @@ pub fn fake_token_stream_for_crate(psess: &ParseSess, krate: &ast::Crate) -> Tok
157157
}
158158

159159
pub fn parse_cfg_attr(
160-
attr: &Attribute,
160+
cfg_attr: &Attribute,
161161
psess: &ParseSess,
162162
) -> Option<(MetaItem, Vec<(AttrItem, Span)>)> {
163163
const CFG_ATTR_GRAMMAR_HELP: &str = "#[cfg_attr(condition, attribute, other_attribute, ...)]";
164164
const CFG_ATTR_NOTE_REF: &str = "for more information, visit \
165165
<https://doc.rust-lang.org/reference/conditional-compilation.html#the-cfg_attr-attribute>";
166166

167-
match attr.get_normal_item().args {
167+
match cfg_attr.get_normal_item().args {
168168
ast::AttrArgs::Delimited(ast::DelimArgs { dspan, delim, ref tokens })
169169
if !tokens.is_empty() =>
170170
{
@@ -180,7 +180,7 @@ pub fn parse_cfg_attr(
180180
}
181181
_ => {
182182
psess.dcx().emit_err(errors::MalformedCfgAttr {
183-
span: attr.span,
183+
span: cfg_attr.span,
184184
sugg: CFG_ATTR_GRAMMAR_HELP,
185185
});
186186
}

0 commit comments

Comments
 (0)