@@ -633,6 +633,8 @@ const tokenize = (input, pos = 0, callbacks = {}) => {
633633 // Tracks whether the current tag has parsed any attributes — used to
634634 // fire the `end-tag-with-attributes` parse error when an end tag emits.
635635 let tagHasAttributes = false;
636+ // Whether the last `<?` stays a processing instruction until EOF would cut it.
637+ let processingInstructionTargetValid = false;
636638 // Memoized next `<` / `&` / NUL for the text-run fast-forwards, where native
637639 // `indexOf` beats a per-char loop. `pos` only moves forward, so each memo is
638640 // refreshed at most once per occurrence; `len` means no further one.
@@ -707,6 +709,68 @@ const tokenize = (input, pos = 0, callbacks = {}) => {
707709 }
708710 };
709711
712+ /**
713+ * Raises the parse errors of the processing instruction open and target
714+ * states for a `<?`, whose token spans what a bogus comment's does: up to the
715+ * first `>`. An EOF one is left to the end of input, so errors stay in order.
716+ * @param {number} targetStart offset past the `?`
717+ * @returns {boolean} true when the target is one a processing instruction keeps
718+ */
719+ const scanProcessingInstructionTarget = (targetStart) => {
720+ if (targetStart >= len) return true;
721+ let c = input.charCodeAt(targetStart);
722+ if (!isAsciiAlpha(c) && c !== CC_LOW_LINE) {
723+ reportError(
724+ "invalid-first-character-of-processing-instruction-target",
725+ targetStart,
726+ targetStart + 1,
727+ "warning"
728+ );
729+ return false;
730+ }
731+ for (let i = targetStart + 1; i < len; i++) {
732+ c = input.charCodeAt(i);
733+ if (
734+ isAsciiWhitespace(c) ||
735+ c === CC_QUESTION_MARK ||
736+ c === CC_GREATER_THAN
737+ ) {
738+ if (
739+ !rangeAsciiCaseInsensitiveMatch(input, targetStart, i, "xml") &&
740+ !rangeAsciiCaseInsensitiveMatch(
741+ input,
742+ targetStart,
743+ i,
744+ "xml-stylesheet"
745+ )
746+ ) {
747+ return true;
748+ }
749+ reportError(
750+ "disallowed-processing-instruction-target",
751+ targetStart,
752+ i,
753+ "warning"
754+ );
755+ return false;
756+ }
757+ if (
758+ !isAsciiAlphanumeric(c) &&
759+ c !== CC_HYPHEN_MINUS &&
760+ c !== CC_LOW_LINE
761+ ) {
762+ reportError(
763+ "invalid-processing-instruction-target",
764+ i,
765+ i + 1,
766+ "warning"
767+ );
768+ return false;
769+ }
770+ }
771+ return true;
772+ };
773+
710774 // Content mode for the tag just opened, named at `lastOpenTagStart..End`. In
711775 // foreign content `isForeign` vetoes the RAWTEXT/RCDATA/script switch, so an
712776 // SVG `<title>` is markup; `lastOpenTagName` is materialized only there.
@@ -802,11 +866,11 @@ const tokenize = (input, pos = 0, callbacks = {}) => {
802866 selfClosing
803867 );
804868 }
805- if (!selfClosing) {
806- // Record offsets only; `contentModeAfterOpenTag` lowercases lazily.
807- lastOpenTagStart = tagNameStart;
808- lastOpenTagEnd = tagNameEnd ;
809- }
869+ // Record offsets only; `contentModeAfterOpenTag` lowercases lazily. A
870+ // trailing `/` is no exception: no element that switches the content mode
871+ // is void, so the tree builder ignores the flag on one in HTML content.
872+ lastOpenTagStart = tagNameStart ;
873+ lastOpenTagEnd = tagNameEnd;
810874 tagHasAttributes = false;
811875 textStart = nextPos;
812876 return nextPos;
@@ -841,7 +905,7 @@ const tokenize = (input, pos = 0, callbacks = {}) => {
841905
842906 // Every WHATWG tokenizer state is handled. Omitted: the parse errors
843907 // needing state an offset scanner lacks (duplicate-attribute,
844- // cdata-in-html-content, `*-in-input-stream`) and reference substitution.
908+ // `*-in-input-stream`) and reference substitution.
845909 switch (state) {
846910 // https://html.spec.whatwg.org/multipage/parsing.html#data-state
847911 case STATE_DATA:
@@ -940,13 +1004,10 @@ const tokenize = (input, pos = 0, callbacks = {}) => {
9401004 pos++;
9411005 }
9421006 } else if (cc === CC_QUESTION_MARK) {
943- // `?`: unexpected-question-mark-instead-of-tag-name parse error. Create a
944- // comment token with empty data, reconsume in bogus comment.
945- reportError(
946- "unexpected-question-mark-instead-of-tag-name",
947- pos,
948- pos + 1,
949- "warning"
1007+ // `?`: the processing instruction states. The `<?…>` is reported as
1008+ // the bogus comment it spans, and the tree tells the two apart.
1009+ processingInstructionTargetValid = scanProcessingInstructionTarget(
1010+ pos + 1
9501011 );
9511012 flushText(tagStart);
9521013 commentStart = tagStart;
@@ -1432,8 +1493,9 @@ const tokenize = (input, pos = 0, callbacks = {}) => {
14321493 state = STATE_DATA;
14331494 pos = emitCloseTag(pos + 1);
14341495 } else {
1435- pos = emitOpenTag(pos + 1, true);
1436- state = STATE_DATA;
1496+ const nextPos = emitOpenTag(pos + 1, true);
1497+ state = nextPos > pos + 1 ? STATE_DATA : contentModeAfterOpenTag();
1498+ pos = nextPos;
14371499 }
14381500 } else {
14391501 // Anything else
@@ -1482,22 +1544,25 @@ const tokenize = (input, pos = 0, callbacks = {}) => {
14821544 } else if (
14831545 // `[CDATA[`: consume it and switch to CDATA section, but only with an
14841546 // adjusted current node outside the HTML namespace. Elsewhere it is a
1485- // bogus comment, leaving the rest as text .
1547+ // bogus comment whose data starts with the `[CDATA[` .
14861548 cc === CC_LEFT_SQUARE_BRACKET &&
14871549 input.charCodeAt(pos + 1) === 0x43 /* C */ &&
14881550 input.charCodeAt(pos + 2) === 0x44 /* D */ &&
14891551 input.charCodeAt(pos + 3) === 0x41 /* A */ &&
14901552 input.charCodeAt(pos + 4) === 0x54 /* T */ &&
14911553 input.charCodeAt(pos + 5) === 0x41 /* A */ &&
1492- input.charCodeAt(pos + 6) === CC_LEFT_SQUARE_BRACKET &&
1493- callbacks.isForeign !== undefined &&
1494- callbacks.isForeign()
1554+ input.charCodeAt(pos + 6) === CC_LEFT_SQUARE_BRACKET
14951555 ) {
1496- pos += 7;
14971556 commentStart = tagStart;
1557+ if (callbacks.isForeign !== undefined && callbacks.isForeign()) {
1558+ pos += 7;
1559+ state = STATE_CDATA_SECTION;
1560+ } else {
1561+ reportError("cdata-in-html-content", tagStart, pos + 7, "warning");
1562+ state = STATE_BOGUS_COMMENT;
1563+ }
14981564 commentDataStart = pos;
14991565 commentDataEnd = pos;
1500- state = STATE_CDATA_SECTION;
15011566 } else {
15021567 // Anything else: incorrectly-opened-comment parse error. Create a comment
15031568 // token with empty data, switch to bogus comment (don't consume anything
@@ -3977,6 +4042,8 @@ const tokenize = (input, pos = 0, callbacks = {}) => {
39774042 if (tagNameStart !== -1 && tagNameEnd < tagNameStart) {
39784043 tagNameEnd = len;
39794044 }
4045+ // The spec drops the end tag EOF cut, so it never has attributes to report.
4046+ tagHasAttributes = false;
39804047 flushText(tagStart);
39814048 pos =
39824049 input.charCodeAt(tagStart + 1) === CC_SOLIDUS
@@ -3998,6 +4065,11 @@ const tokenize = (input, pos = 0, callbacks = {}) => {
39984065 commentDataEnd = len;
39994066 } else if (state !== STATE_BOGUS_COMMENT) {
40004067 reportError("eof-in-comment", len, len, "error");
4068+ } else if (
4069+ processingInstructionTargetValid &&
4070+ input.charCodeAt(commentStart + 1) === CC_QUESTION_MARK
4071+ ) {
4072+ reportError("eof-in-processing-instruction", len, len, "error");
40014073 }
40024074 if (callbacks.comment !== undefined) {
40034075 pos = callbacks.comment(
@@ -4036,19 +4108,12 @@ const tokenize = (input, pos = 0, callbacks = {}) => {
40364108 }
40374109 } else {
40384110 if (
4039- state === STATE_SCRIPT_DATA_ESCAPED ||
4040- state === STATE_SCRIPT_DATA_ESCAPED_DASH ||
4041- state === STATE_SCRIPT_DATA_ESCAPED_DASH_DASH ||
4042- state === STATE_SCRIPT_DATA_ESCAPED_LESS_THAN_SIGN ||
4043- state === STATE_SCRIPT_DATA_DOUBLE_ESCAPED ||
4044- state === STATE_SCRIPT_DATA_DOUBLE_ESCAPED_DASH ||
4045- state === STATE_SCRIPT_DATA_DOUBLE_ESCAPED_DASH_DASH ||
4046- state === STATE_SCRIPT_DATA_DOUBLE_ESCAPED_LESS_THAN_SIGN ||
4047- state === STATE_SCRIPT_DATA_DOUBLE_ESCAPE_END
4111+ state >= STATE_SCRIPT_DATA_ESCAPED &&
4112+ state <= STATE_SCRIPT_DATA_DOUBLE_ESCAPE_END
40484113 ) {
4049- // Inside `<script><!-- … ` at EOF: an
4050- // eof-in-script-html-comment-like-text parse error. The less-than-sign
4051- // and double-escape-end states reconsume into it at EOF, per spec.
4114+ // Inside `<script><!-- … ` at EOF: eof-in-script-html-comment-like-text.
4115+ // The escaped end tag name state is a tag above; the others reconsume
4116+ // into an escaped or double-escaped state at EOF, per spec.
40524117 reportError("eof-in-script-html-comment-like-text", len, len, "error");
40534118 } else if (state === STATE_TAG_OPEN || state === STATE_END_TAG_OPEN) {
40544119 // `<` or `</` with nothing after; spec calls this
0 commit comments