Skip to content

Commit 2018e5e

Browse files
alexander-akaitalexander-akait
committed
fix(javascript): read /= in expression position as a regexp
- Where an expression belongs, a `/=` the tokenizer took for the division-assignment operator now re-reads as a regexp, as a lone `/` already did: `await /=a/g`, `x\n{} /=a/g` and the like parse, as the spec, V8, meriyah and Babel read them (acorn rejects them). - WebpackParser's lazy template path raised a TypeError for a `${…}` closed at the end of input; it now raises acorn's "Unterminated template literal" at the same position. Checked against the lexical grammar with differential fuzzing against acorn, V8, meriyah and Babel. Claude-Session: https://claude.ai/code/session_017TkBWTk3wcyUcJYDp5ZhaP
1 parent 9e8cb55 commit 2018e5e

3 files changed

Lines changed: 51 additions & 6 deletions

File tree

‎.changeset/060-tokenizer-fixes.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,4 +2,4 @@
22
"webpack": patch
33
---
44

5-
Fix CSS and JavaScript tokenizer edge cases.
5+
Fix CSS, HTML and JavaScript tokenizer divergences from their specs.

‎lib/javascript/syntax-parser.js‎

Lines changed: 19 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -5020,9 +5020,14 @@ class SyntaxParser {
50205020
* @returns {Expression | NodeOf<"Super">} the expression
50215021
*/
50225022
parseExprAtom(refDestructuringErrors, forInit, forNew) {
5023-
// A `/` where an expression belongs means the tokenizer guessed division,
5024-
// so make it read the regexp it really is.
5025-
if (this.type === tokTypes.slash) this.readRegexp();
5023+
// A `/` or `/=` where an expression belongs means the tokenizer guessed
5024+
// division, so make it read the regexp it really is.
5025+
if (this.type === tokTypes.slash) {
5026+
this.readRegexp();
5027+
} else if (this.type === tokTypes.assign && this.value === "/=") {
5028+
this.pos = this.start + 1;
5029+
this.readRegexp();
5030+
}
50265031

50275032
let node;
50285033
const canBeArrow = this.potentialArrowAt === this.start;
@@ -15373,6 +15378,10 @@ class WebpackParser extends BaseParser {
1537315378
cooked: null
1537415379
};
1537515380
} else {
15381+
// `${…}` closed at the end of input reads eof, which parseTemplate reports
15382+
if (this.type === tokTypes.eof) {
15383+
this.raise(this.pos, "Unterminated template literal");
15384+
}
1537615385
const cooked = /** @type {string} */ (this.value);
1537715386
// every escape and CRLF cook shorter and a lone CR cooks to the LF raw
1537815387
// normalization yields: an equal-length cooked string IS the normalized raw
@@ -15864,8 +15873,13 @@ class WebpackParser extends BaseParser {
1586415873
);
1586515874
}
1586615875
// division in expression position: the tokenizer got confused, force a
15867-
// regexp re-read (mirrors the top of acorn's parseExprAtom)
15868-
if (this.type === tokTypes.slash) this.readRegexp();
15876+
// regexp re-read (acorn's misses `/=`, as in `await /=a/`)
15877+
if (this.type === tokTypes.slash) {
15878+
this.readRegexp();
15879+
} else if (this.type === tokTypes.assign && this.value === "/=") {
15880+
this.pos = this.start + 1;
15881+
this.readRegexp();
15882+
}
1586915883
const type = this.type;
1587015884
if (type === tokTypes.name) {
1587115885
const canBeArrow = this.potentialArrowAt === this.start;

‎test/unitCases/javascript/tokenizer.unittest.js‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,32 @@ describe("WebpackParser", () => {
9191
expect(parse("a.in / b; a.of / c;").ast).toBeDefined();
9292
});
9393

94+
it("should re-read a `/=` the tokenizer took for division as a regexp", () => {
95+
// `await` and a block's `}` leave the tokenizer guessing division, but
96+
// an expression belongs there, so the spec reads a regexp (acorn throws)
97+
const sources = [
98+
"async function f() { await /=a/g; }",
99+
"x\n{} /=a/g",
100+
"class A { static { class B {} /=a/g; } }"
101+
];
102+
for (const source of sources) {
103+
expect(() => acorn.parse(source, { ecmaVersion: "latest" })).toThrow(
104+
/Unexpected token/
105+
);
106+
for (const options of [{}, { locations: true }]) {
107+
expect(JSON.stringify(parse(source, options).ast)).toContain(
108+
'"pattern":"=a"'
109+
);
110+
}
111+
expect(
112+
JSON.stringify(Parser.parse(source, { ecmaVersion: "latest" }))
113+
).toContain('"pattern":"=a"');
114+
}
115+
expect(() => parse("x = /=")).toThrow(
116+
/Unterminated regular expression \(1:5\)/
117+
);
118+
});
119+
94120
it("should re-allow an expression after `of` and generator `yield`", () => {
95121
// the inlined name.updateContext true branches: the `/` lexes a regexp
96122
expect(parse("for (x of /re/g);").ast).toBeDefined();
@@ -219,6 +245,11 @@ describe("WebpackParser", () => {
219245
);
220246
// a cold template chunk (escape seen) that never closes is unterminated
221247
expect(() => parse("`a\\t")).toThrow(/Unterminated template/);
248+
// input that ends right after a substitution's `}` is a SyntaxError
249+
// like acorn's, not a TypeError from the lazy template element
250+
expect(() => parse("x = `${ 1 }")).toThrow(
251+
/Unterminated template literal \(1:11\)/
252+
);
222253
});
223254

224255
it("should read radix integers and bigints", () => {

0 commit comments

Comments
 (0)