Skip to content

Commit fcef46c

Browse files
charliermarshcharliermarsh
authored
Fix %foo? parsing in IPython assignment expressions (#24152)
## Summary We now distinguish between IPython escape commands lexed after an `=` sign from those at the start of a logical line, which allows `x = %foo?` to be interpreted as "assign the result of running a line magic named `foo?`", matching the IPython assignment-magic transform. See: #21705 (comment).
1 parent 9d2b160 commit fcef46c

5 files changed

Lines changed: 153 additions & 19 deletions

‎crates/ruff_python_parser/src/lexer.rs‎

Lines changed: 46 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -434,7 +434,10 @@ impl<'src> Lexer<'src> {
434434
&& self.nesting == 0 =>
435435
{
436436
// SAFETY: Safe because `c` has been matched against one of the possible escape command token
437-
self.lex_ipython_escape_command(IpyEscapeKind::try_from(c).unwrap())
437+
self.lex_ipython_escape_command(
438+
IpyEscapeKind::try_from(c).unwrap(),
439+
IpyEscapeLexContext::Assignment,
440+
)
438441
}
439442

440443
c @ ('%' | '!' | '?' | '/' | ';' | ',')
@@ -448,7 +451,7 @@ impl<'src> Lexer<'src> {
448451
IpyEscapeKind::try_from(c).unwrap()
449452
};
450453

451-
self.lex_ipython_escape_command(kind)
454+
self.lex_ipython_escape_command(kind, IpyEscapeLexContext::LogicalLineStart)
452455
}
453456

454457
'?' if self.mode == Mode::Ipython => TokenKind::Question,
@@ -1262,7 +1265,11 @@ impl<'src> Lexer<'src> {
12621265
}
12631266

12641267
/// Lex a single IPython escape command.
1265-
fn lex_ipython_escape_command(&mut self, escape_kind: IpyEscapeKind) -> TokenKind {
1268+
fn lex_ipython_escape_command(
1269+
&mut self,
1270+
escape_kind: IpyEscapeKind,
1271+
context: IpyEscapeLexContext,
1272+
) -> TokenKind {
12661273
let mut value = String::new();
12671274

12681275
loop {
@@ -1310,6 +1317,27 @@ impl<'src> Lexer<'src> {
13101317
question_count += 1;
13111318
}
13121319

1320+
// Help end tokens (`?` / `??`) are only valid in certain contexts
1321+
// (e.g., not within f-strings or parenthesized expressions), and only
1322+
// for escape kinds that IPython recognizes as supporting a trailing `?`
1323+
// (i.e., `%`, `%%`, `?`, and `??`). For other escape kinds like `!` or
1324+
// `/`, the `?` is just part of the command value.
1325+
if !context.allows_help_end()
1326+
|| !matches!(
1327+
escape_kind,
1328+
IpyEscapeKind::Magic
1329+
| IpyEscapeKind::Magic2
1330+
| IpyEscapeKind::Help
1331+
| IpyEscapeKind::Help2
1332+
)
1333+
{
1334+
value.reserve(question_count as usize);
1335+
for _ in 0..question_count {
1336+
value.push('?');
1337+
}
1338+
continue;
1339+
}
1340+
13131341
// The original implementation in the IPython codebase is based on regex which
13141342
// means that it's strict in the sense that it won't recognize a help end escape:
13151343
// * If there's any whitespace before the escape token (e.g. `%foo ?`)
@@ -1748,6 +1776,18 @@ impl State {
17481776
}
17491777
}
17501778

1779+
#[derive(Copy, Clone, Debug)]
1780+
enum IpyEscapeLexContext {
1781+
Assignment,
1782+
LogicalLineStart,
1783+
}
1784+
1785+
impl IpyEscapeLexContext {
1786+
const fn allows_help_end(self) -> bool {
1787+
matches!(self, Self::LogicalLineStart)
1788+
}
1789+
}
1790+
17511791
#[derive(Copy, Clone, Debug)]
17521792
enum Radix {
17531793
Binary,
@@ -2108,7 +2148,9 @@ pwd = !pwd
21082148
foo = %timeit a = b
21092149
bar = %timeit a % 3
21102150
baz = %matplotlib \
2111-
inline"
2151+
inline
2152+
qux = %foo?
2153+
quux = !pwd?"
21122154
.trim();
21132155
assert_snapshot!(lex_jupyter_source(source));
21142156
}

‎crates/ruff_python_parser/src/parser/snapshots/ruff_python_parser__parser__tests__ipython_escape_commands.snap‎

Lines changed: 60 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ expression: parsed.syntax()
55
Module(
66
ModModule {
77
node_index: NodeIndex(None),
8-
range: 0..929,
8+
range: 0..953,
99
body: [
1010
Expr(
1111
StmtExpr {
@@ -362,23 +362,71 @@ Module(
362362
),
363363
},
364364
),
365+
Assign(
366+
StmtAssign {
367+
node_index: NodeIndex(None),
368+
range: 785..796,
369+
targets: [
370+
Name(
371+
ExprName {
372+
node_index: NodeIndex(None),
373+
range: 785..788,
374+
id: Name("bar"),
375+
ctx: Store,
376+
},
377+
),
378+
],
379+
value: IpyEscapeCommand(
380+
ExprIpyEscapeCommand {
381+
node_index: NodeIndex(None),
382+
range: 791..796,
383+
kind: Magic,
384+
value: "foo?",
385+
},
386+
),
387+
},
388+
),
389+
Assign(
390+
StmtAssign {
391+
node_index: NodeIndex(None),
392+
range: 797..808,
393+
targets: [
394+
Name(
395+
ExprName {
396+
node_index: NodeIndex(None),
397+
range: 797..800,
398+
id: Name("baz"),
399+
ctx: Store,
400+
},
401+
),
402+
],
403+
value: IpyEscapeCommand(
404+
ExprIpyEscapeCommand {
405+
node_index: NodeIndex(None),
406+
range: 803..808,
407+
kind: Shell,
408+
value: "pwd?",
409+
},
410+
),
411+
},
412+
),
365413
IpyEscapeCommand(
366414
StmtIpyEscapeCommand {
367415
node_index: NodeIndex(None),
368-
range: 786..791,
416+
range: 810..815,
369417
kind: Magic,
370418
value: " foo",
371419
},
372420
),
373421
Assign(
374422
StmtAssign {
375423
node_index: NodeIndex(None),
376-
range: 792..813,
424+
range: 816..837,
377425
targets: [
378426
Name(
379427
ExprName {
380428
node_index: NodeIndex(None),
381-
range: 792..795,
429+
range: 816..819,
382430
id: Name("foo"),
383431
ctx: Store,
384432
},
@@ -387,7 +435,7 @@ Module(
387435
value: IpyEscapeCommand(
388436
ExprIpyEscapeCommand {
389437
node_index: NodeIndex(None),
390-
range: 798..813,
438+
range: 822..837,
391439
kind: Magic,
392440
value: "foo # comment",
393441
},
@@ -397,55 +445,55 @@ Module(
397445
IpyEscapeCommand(
398446
StmtIpyEscapeCommand {
399447
node_index: NodeIndex(None),
400-
range: 838..842,
448+
range: 862..866,
401449
kind: Help,
402450
value: "foo",
403451
},
404452
),
405453
IpyEscapeCommand(
406454
StmtIpyEscapeCommand {
407455
node_index: NodeIndex(None),
408-
range: 843..852,
456+
range: 867..876,
409457
kind: Help2,
410458
value: "foo.bar",
411459
},
412460
),
413461
IpyEscapeCommand(
414462
StmtIpyEscapeCommand {
415463
node_index: NodeIndex(None),
416-
range: 853..865,
464+
range: 877..889,
417465
kind: Help,
418466
value: "foo.bar.baz",
419467
},
420468
),
421469
IpyEscapeCommand(
422470
StmtIpyEscapeCommand {
423471
node_index: NodeIndex(None),
424-
range: 866..874,
472+
range: 890..898,
425473
kind: Help2,
426474
value: "foo[0]",
427475
},
428476
),
429477
IpyEscapeCommand(
430478
StmtIpyEscapeCommand {
431479
node_index: NodeIndex(None),
432-
range: 875..885,
480+
range: 899..909,
433481
kind: Help,
434482
value: "foo[0][1]",
435483
},
436484
),
437485
IpyEscapeCommand(
438486
StmtIpyEscapeCommand {
439487
node_index: NodeIndex(None),
440-
range: 886..905,
488+
range: 910..929,
441489
kind: Help2,
442490
value: "foo.bar[0].baz[1]",
443491
},
444492
),
445493
IpyEscapeCommand(
446494
StmtIpyEscapeCommand {
447495
node_index: NodeIndex(None),
448-
range: 906..929,
496+
range: 930..953,
449497
kind: Help2,
450498
value: "foo.bar[0].baz[2].egg",
451499
},

‎crates/ruff_python_parser/src/parser/tests.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -115,6 +115,8 @@ p1 = !pwd
115115
p2: str = !pwd
116116
foo = %foo \
117117
bar
118+
bar = %foo?
119+
baz = !pwd?
118120
119121
% foo
120122
foo = %foo # comment

‎crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__ipython_escape_command_assignment.snap‎

Lines changed: 43 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -87,7 +87,49 @@ expression: lex_jupyter_source(source)
8787
),
8888
(
8989
Newline,
90-
85..85,
90+
85..86,
91+
),
92+
(
93+
Name(
94+
Name("qux"),
95+
),
96+
86..89,
97+
),
98+
(
99+
Equal,
100+
90..91,
101+
),
102+
(
103+
IpyEscapeCommand {
104+
value: "foo?",
105+
kind: Magic,
106+
},
107+
92..97,
108+
),
109+
(
110+
Newline,
111+
97..98,
112+
),
113+
(
114+
Name(
115+
Name("quux"),
116+
),
117+
98..102,
118+
),
119+
(
120+
Equal,
121+
103..104,
122+
),
123+
(
124+
IpyEscapeCommand {
125+
value: "pwd?",
126+
kind: Shell,
127+
},
128+
105..110,
129+
),
130+
(
131+
Newline,
132+
110..110,
91133
),
92134
]
93135
```

‎crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__ipython_help_end_escape_command.snap‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -172,8 +172,8 @@ expression: lex_jupyter_source(source)
172172
),
173173
(
174174
IpyEscapeCommand {
175-
value: "pwd",
176-
kind: Help,
175+
value: "pwd?",
176+
kind: Shell,
177177
},
178178
127..132,
179179
),

0 commit comments

Comments
 (0)