Skip to content

Commit 9e8cb55

Browse files
fix(html): tokenize like the WHATWG tokenizer in five more places
- `<?…>` reports the processing-instruction errors the standard now defines (none for a well-formed one; invalid, disallowed or missing targets; EOF) instead of the retired unexpected-question-mark-instead-of-tag-name; it still reads as one bogus comment. - `<![CDATA[` outside foreign content is cdata-in-html-content. - An end tag cut off by EOF reports eof-in-tag alone, not also end-tag-with-attributes. - EOF in the script's escaped end-tag-open and double-escape-start states reports eof-in-script-html-comment-like-text. - A self-closing `<title/>`, `<script/>`, `<style/>` or `<noframes/>` switches the content mode the tree builder already switched to, so `<title/><b></title>` keeps `<b>` as text. Checked against the html5lib tokenizer tests (tokens and errors), parse5's tokenizer and the entities package. Claude-Session: https://claude.ai/code/session_017TkBWTk3wcyUcJYDp5ZhaP
1 parent 820e700 commit 9e8cb55

2 files changed

Lines changed: 215 additions & 42 deletions

File tree

‎lib/html/syntax-parser.js‎

Lines changed: 98 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -633,6 +633,8 @@ const tokenize = (input, pos = 0, callbacks = {}) => {
633633
// Tracks whether the current tag has parsed any attributes — used to
634634
// fire the `end-tag-with-attributes` parse error when an end tag emits.
635635
let tagHasAttributes = false;
636+
// Whether the last `<?` stays a processing instruction until EOF would cut it.
637+
let processingInstructionTargetValid = false;
636638
// Memoized next `<` / `&` / NUL for the text-run fast-forwards, where native
637639
// `indexOf` beats a per-char loop. `pos` only moves forward, so each memo is
638640
// refreshed at most once per occurrence; `len` means no further one.
@@ -707,6 +709,68 @@ const tokenize = (input, pos = 0, callbacks = {}) => {
707709
}
708710
};
709711

712+
/**
713+
* Raises the parse errors of the processing instruction open and target
714+
* states for a `<?`, whose token spans what a bogus comment's does: up to the
715+
* first `>`. An EOF one is left to the end of input, so errors stay in order.
716+
* @param {number} targetStart offset past the `?`
717+
* @returns {boolean} true when the target is one a processing instruction keeps
718+
*/
719+
const scanProcessingInstructionTarget = (targetStart) => {
720+
if (targetStart >= len) return true;
721+
let c = input.charCodeAt(targetStart);
722+
if (!isAsciiAlpha(c) && c !== CC_LOW_LINE) {
723+
reportError(
724+
"invalid-first-character-of-processing-instruction-target",
725+
targetStart,
726+
targetStart + 1,
727+
"warning"
728+
);
729+
return false;
730+
}
731+
for (let i = targetStart + 1; i < len; i++) {
732+
c = input.charCodeAt(i);
733+
if (
734+
isAsciiWhitespace(c) ||
735+
c === CC_QUESTION_MARK ||
736+
c === CC_GREATER_THAN
737+
) {
738+
if (
739+
!rangeAsciiCaseInsensitiveMatch(input, targetStart, i, "xml") &&
740+
!rangeAsciiCaseInsensitiveMatch(
741+
input,
742+
targetStart,
743+
i,
744+
"xml-stylesheet"
745+
)
746+
) {
747+
return true;
748+
}
749+
reportError(
750+
"disallowed-processing-instruction-target",
751+
targetStart,
752+
i,
753+
"warning"
754+
);
755+
return false;
756+
}
757+
if (
758+
!isAsciiAlphanumeric(c) &&
759+
c !== CC_HYPHEN_MINUS &&
760+
c !== CC_LOW_LINE
761+
) {
762+
reportError(
763+
"invalid-processing-instruction-target",
764+
i,
765+
i + 1,
766+
"warning"
767+
);
768+
return false;
769+
}
770+
}
771+
return true;
772+
};
773+
710774
// Content mode for the tag just opened, named at `lastOpenTagStart..End`. In
711775
// foreign content `isForeign` vetoes the RAWTEXT/RCDATA/script switch, so an
712776
// SVG `<title>` is markup; `lastOpenTagName` is materialized only there.
@@ -802,11 +866,11 @@ const tokenize = (input, pos = 0, callbacks = {}) => {
802866
selfClosing
803867
);
804868
}
805-
if (!selfClosing) {
806-
// Record offsets only; `contentModeAfterOpenTag` lowercases lazily.
807-
lastOpenTagStart = tagNameStart;
808-
lastOpenTagEnd = tagNameEnd;
809-
}
869+
// Record offsets only; `contentModeAfterOpenTag` lowercases lazily. A
870+
// trailing `/` is no exception: no element that switches the content mode
871+
// is void, so the tree builder ignores the flag on one in HTML content.
872+
lastOpenTagStart = tagNameStart;
873+
lastOpenTagEnd = tagNameEnd;
810874
tagHasAttributes = false;
811875
textStart = nextPos;
812876
return nextPos;
@@ -841,7 +905,7 @@ const tokenize = (input, pos = 0, callbacks = {}) => {
841905

842906
// Every WHATWG tokenizer state is handled. Omitted: the parse errors
843907
// needing state an offset scanner lacks (duplicate-attribute,
844-
// cdata-in-html-content, `*-in-input-stream`) and reference substitution.
908+
// `*-in-input-stream`) and reference substitution.
845909
switch (state) {
846910
// https://html.spec.whatwg.org/multipage/parsing.html#data-state
847911
case STATE_DATA:
@@ -940,13 +1004,10 @@ const tokenize = (input, pos = 0, callbacks = {}) => {
9401004
pos++;
9411005
}
9421006
} else if (cc === CC_QUESTION_MARK) {
943-
// `?`: unexpected-question-mark-instead-of-tag-name parse error. Create a
944-
// comment token with empty data, reconsume in bogus comment.
945-
reportError(
946-
"unexpected-question-mark-instead-of-tag-name",
947-
pos,
948-
pos + 1,
949-
"warning"
1007+
// `?`: the processing instruction states. The `<?…>` is reported as
1008+
// the bogus comment it spans, and the tree tells the two apart.
1009+
processingInstructionTargetValid = scanProcessingInstructionTarget(
1010+
pos + 1
9501011
);
9511012
flushText(tagStart);
9521013
commentStart = tagStart;
@@ -1432,8 +1493,9 @@ const tokenize = (input, pos = 0, callbacks = {}) => {
14321493
state = STATE_DATA;
14331494
pos = emitCloseTag(pos + 1);
14341495
} else {
1435-
pos = emitOpenTag(pos + 1, true);
1436-
state = STATE_DATA;
1496+
const nextPos = emitOpenTag(pos + 1, true);
1497+
state = nextPos > pos + 1 ? STATE_DATA : contentModeAfterOpenTag();
1498+
pos = nextPos;
14371499
}
14381500
} else {
14391501
// Anything else
@@ -1482,22 +1544,25 @@ const tokenize = (input, pos = 0, callbacks = {}) => {
14821544
} else if (
14831545
// `[CDATA[`: consume it and switch to CDATA section, but only with an
14841546
// adjusted current node outside the HTML namespace. Elsewhere it is a
1485-
// bogus comment, leaving the rest as text.
1547+
// bogus comment whose data starts with the `[CDATA[`.
14861548
cc === CC_LEFT_SQUARE_BRACKET &&
14871549
input.charCodeAt(pos + 1) === 0x43 /* C */ &&
14881550
input.charCodeAt(pos + 2) === 0x44 /* D */ &&
14891551
input.charCodeAt(pos + 3) === 0x41 /* A */ &&
14901552
input.charCodeAt(pos + 4) === 0x54 /* T */ &&
14911553
input.charCodeAt(pos + 5) === 0x41 /* A */ &&
1492-
input.charCodeAt(pos + 6) === CC_LEFT_SQUARE_BRACKET &&
1493-
callbacks.isForeign !== undefined &&
1494-
callbacks.isForeign()
1554+
input.charCodeAt(pos + 6) === CC_LEFT_SQUARE_BRACKET
14951555
) {
1496-
pos += 7;
14971556
commentStart = tagStart;
1557+
if (callbacks.isForeign !== undefined && callbacks.isForeign()) {
1558+
pos += 7;
1559+
state = STATE_CDATA_SECTION;
1560+
} else {
1561+
reportError("cdata-in-html-content", tagStart, pos + 7, "warning");
1562+
state = STATE_BOGUS_COMMENT;
1563+
}
14981564
commentDataStart = pos;
14991565
commentDataEnd = pos;
1500-
state = STATE_CDATA_SECTION;
15011566
} else {
15021567
// Anything else: incorrectly-opened-comment parse error. Create a comment
15031568
// token with empty data, switch to bogus comment (don't consume anything
@@ -3977,6 +4042,8 @@ const tokenize = (input, pos = 0, callbacks = {}) => {
39774042
if (tagNameStart !== -1 && tagNameEnd < tagNameStart) {
39784043
tagNameEnd = len;
39794044
}
4045+
// The spec drops the end tag EOF cut, so it never has attributes to report.
4046+
tagHasAttributes = false;
39804047
flushText(tagStart);
39814048
pos =
39824049
input.charCodeAt(tagStart + 1) === CC_SOLIDUS
@@ -3998,6 +4065,11 @@ const tokenize = (input, pos = 0, callbacks = {}) => {
39984065
commentDataEnd = len;
39994066
} else if (state !== STATE_BOGUS_COMMENT) {
40004067
reportError("eof-in-comment", len, len, "error");
4068+
} else if (
4069+
processingInstructionTargetValid &&
4070+
input.charCodeAt(commentStart + 1) === CC_QUESTION_MARK
4071+
) {
4072+
reportError("eof-in-processing-instruction", len, len, "error");
40014073
}
40024074
if (callbacks.comment !== undefined) {
40034075
pos = callbacks.comment(
@@ -4036,19 +4108,12 @@ const tokenize = (input, pos = 0, callbacks = {}) => {
40364108
}
40374109
} else {
40384110
if (
4039-
state === STATE_SCRIPT_DATA_ESCAPED ||
4040-
state === STATE_SCRIPT_DATA_ESCAPED_DASH ||
4041-
state === STATE_SCRIPT_DATA_ESCAPED_DASH_DASH ||
4042-
state === STATE_SCRIPT_DATA_ESCAPED_LESS_THAN_SIGN ||
4043-
state === STATE_SCRIPT_DATA_DOUBLE_ESCAPED ||
4044-
state === STATE_SCRIPT_DATA_DOUBLE_ESCAPED_DASH ||
4045-
state === STATE_SCRIPT_DATA_DOUBLE_ESCAPED_DASH_DASH ||
4046-
state === STATE_SCRIPT_DATA_DOUBLE_ESCAPED_LESS_THAN_SIGN ||
4047-
state === STATE_SCRIPT_DATA_DOUBLE_ESCAPE_END
4111+
state >= STATE_SCRIPT_DATA_ESCAPED &&
4112+
state <= STATE_SCRIPT_DATA_DOUBLE_ESCAPE_END
40484113
) {
4049-
// Inside `<script><!-- … ` at EOF: an
4050-
// eof-in-script-html-comment-like-text parse error. The less-than-sign
4051-
// and double-escape-end states reconsume into it at EOF, per spec.
4114+
// Inside `<script><!-- … ` at EOF: eof-in-script-html-comment-like-text.
4115+
// The escaped end tag name state is a tag above; the others reconsume
4116+
// into an escaped or double-escaped state at EOF, per spec.
40524117
reportError("eof-in-script-html-comment-like-text", len, len, "error");
40534118
} else if (state === STATE_TAG_OPEN || state === STATE_END_TAG_OPEN) {
40544119
// `<` or `</` with nothing after; spec calls this

‎test/unitCases/html/tokenizer.unittest.js‎

Lines changed: 117 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -2283,15 +2283,65 @@ describe("tokenize", () => {
22832283
]);
22842284
});
22852285

2286-
it("reports unexpected-question-mark-instead-of-tag-name as a warning", () => {
2287-
const errors = collectErrors("a<?pi?>b");
2288-
expect(errors).toEqual([
2286+
it("reports no error for a well-formed processing instruction", () => {
2287+
// The obsolete unexpected-question-mark-instead-of-tag-name is gone: `<?`
2288+
// opens the processing instruction states (§13.2.5.42-46).
2289+
expect(collectErrors("a<?pi?>b<?_a-1 x?y>")).toEqual([]);
2290+
// A target cut at `?`, `>` or whitespace may spell `xml` inside a longer name.
2291+
expect(collectErrors("<?xmls?><?xm>")).toEqual([]);
2292+
});
2293+
2294+
it("reports the processing instruction target errors as warnings", () => {
2295+
expect(collectErrors("<?1>")).toEqual([
2296+
{
2297+
code: "invalid-first-character-of-processing-instruction-target",
2298+
slice: "1",
2299+
severity: "warning"
2300+
}
2301+
]);
2302+
expect(collectErrors("<?a.b>")).toEqual([
2303+
{
2304+
code: "invalid-processing-instruction-target",
2305+
slice: ".",
2306+
severity: "warning"
2307+
}
2308+
]);
2309+
expect(collectErrors("<?XML a?><?xml-stylesheet>")).toEqual([
2310+
{
2311+
code: "disallowed-processing-instruction-target",
2312+
slice: "XML",
2313+
severity: "warning"
2314+
},
22892315
{
2290-
code: "unexpected-question-mark-instead-of-tag-name",
2291-
slice: "?",
2316+
code: "disallowed-processing-instruction-target",
2317+
slice: "xml-stylesheet",
22922318
severity: "warning"
22932319
}
22942320
]);
2321+
// A NUL after an invalid target is in the bogus comment that follows.
2322+
expect(collectErrors("<?\0>").map((e) => e.code)).toEqual([
2323+
"invalid-first-character-of-processing-instruction-target",
2324+
"unexpected-null-character"
2325+
]);
2326+
});
2327+
2328+
it("reports eof-in-processing-instruction as an error, after earlier ones", () => {
2329+
for (const html of ["<?", "<?a", "<?a b?", "<?a>x<?"]) {
2330+
expect(collectErrors(html)).toEqual([
2331+
{ code: "eof-in-processing-instruction", slice: "", severity: "error" }
2332+
]);
2333+
}
2334+
// An invalid target makes it a bogus comment, which EOF ends silently.
2335+
expect(collectErrors("<?1").map((e) => e.code)).toEqual([
2336+
"invalid-first-character-of-processing-instruction-target"
2337+
]);
2338+
expect(collectErrors("<?xml ").map((e) => e.code)).toEqual([
2339+
"disallowed-processing-instruction-target"
2340+
]);
2341+
// So does one a `<?` before it does not open.
2342+
expect(collectErrors("<?a>x</1").map((e) => e.code)).toEqual([
2343+
"invalid-first-character-of-tag-name"
2344+
]);
22952345
});
22962346

22972347
it("reports invalid-first-character-of-tag-name as a warning", () => {
@@ -2782,17 +2832,35 @@ describe("tokenize", () => {
27822832
});
27832833

27842834
it("reports an unclosed CDATA outside foreign content as a bogus comment", () => {
2785-
// No adjusted current node, so `<![CDATA[` is not a CDATA section but an
2786-
// incorrectly-opened comment (§13.2.5.42); `eof-in-cdata` is unreachable
2787-
// from here and is covered by the foreign-content path.
2835+
// No adjusted current node, so `<![CDATA[` is not a CDATA section but a
2836+
// cdata-in-html-content bogus comment (§13.2.5.42); `eof-in-cdata` is
2837+
// covered by the foreign-content path.
27882838
const errors = collectErrors("<![CDATA[unclosed");
27892839
// The bogus comment state emits its token at EOF without a further
27902840
// parse error, so this is the only one.
27912841
expect(errors).toEqual([
2792-
{ code: "incorrectly-opened-comment", slice: "<!", severity: "warning" }
2842+
{
2843+
code: "cdata-in-html-content",
2844+
slice: "<![CDATA[",
2845+
severity: "warning"
2846+
}
27932847
]);
27942848
});
27952849

2850+
it("reports cdata-in-html-content where isForeign says HTML content", () => {
2851+
/** @type {string[]} */
2852+
const out = [];
2853+
tokenize("<![CDATA[a]]>", 0, {
2854+
isForeign: () => false,
2855+
comment: (input, start, end, dataStart, dataEnd) => {
2856+
out.push(input.slice(dataStart, dataEnd));
2857+
return end;
2858+
},
2859+
parseError: (input, code) => out.push(code)
2860+
});
2861+
expect(out).toEqual(["cdata-in-html-content", "[CDATA[a]]"]);
2862+
});
2863+
27962864
it("reports eof-in-script-html-comment-like-text as an error", () => {
27972865
const errors = collectErrors("<script><!-- unclosed");
27982866
expect(errors).toEqual([
@@ -2804,6 +2872,28 @@ describe("tokenize", () => {
28042872
]);
28052873
});
28062874

2875+
it("reports eof-in-script-html-comment-like-text from the states that reconsume into it", () => {
2876+
// Escaped end tag open and double escape start emit what they hold and
2877+
// reconsume in the escaped state, whose EOF is the error.
2878+
for (const html of ["<script><!--</", "<script><!--<a", "<script><!--<"]) {
2879+
expect(collectErrors(html)).toEqual([
2880+
{
2881+
code: "eof-in-script-html-comment-like-text",
2882+
slice: "",
2883+
severity: "error"
2884+
}
2885+
]);
2886+
}
2887+
});
2888+
2889+
it("reports no end-tag-with-attributes for an end tag cut by EOF", () => {
2890+
// The spec drops the token, so eof-in-tag is its only error.
2891+
expect(collectErrors("</a b").map((e) => e.code)).toEqual(["eof-in-tag"]);
2892+
expect(collectErrors("</a b=1 ").map((e) => e.code)).toEqual([
2893+
"eof-in-tag"
2894+
]);
2895+
});
2896+
28072897
it("reports eof-before-tag-name as a warning for lone `<`", () => {
28082898
const errors = collectErrors("hello<");
28092899
expect(errors).toEqual([
@@ -3698,6 +3788,24 @@ describe("tokenize — content modes, CDATA and NUL arcs", () => {
36983788
]);
36993789
});
37003790

3791+
it("switches the content mode after a self-closing start tag as well", () => {
3792+
// None of these elements is void, so the tree builder ignores the `/`
3793+
// and switches the tokenizer as it would for `<title>` (§13.2.6.4.4).
3794+
expect(walk("<title/><b></title><style/><b></style>")).toEqual([
3795+
["open", "title"],
3796+
["text", "<b>"],
3797+
["close", "title"],
3798+
["open", "style"],
3799+
["text", "<b>"],
3800+
["close", "style"]
3801+
]);
3802+
// Unless the tree builder vetoes it, as foreign content does.
3803+
expect(walk("<title/><b>", { vetoesContentMode: () => true })).toEqual([
3804+
["open", "title"],
3805+
["open", "b"]
3806+
]);
3807+
});
3808+
37013809
it("keeps the data state after tags sharing a content-mode tag's length", () => {
37023810
// `section`, `fieldset` and `my-widget` have the lengths of `noembed`,
37033811
// `textarea` and `plaintext` but none of their content modes.

0 commit comments

Comments
 (0)