HTML: return text and template tokens separately in lexer

tdewolff · tdewolff · commit 73af9d7350a0 · 2025-04-05T10:56:51.000+02:00
diff --git a/html/lex.go b/html/lex.go
@@ -23,6 +23,7 @@ const (
 	TextToken
 	SvgToken
 	MathToken
+	TemplateToken
 )
 
 // String returns the string representation of a TokenType.
@@ -185,18 +186,19 @@ func (l *Lexer) Next() (TokenType, []byte) {
 			} else if c == '!' {
 				l.r.Move(2)
 				return l.readMarkup()
-			} else if 0 < len(l.tmplBegin) && l.at(l.tmplBegin...) {
-				l.r.Move(len(l.tmplBegin))
-				l.moveTemplate()
-				l.hasTmpl = true
 			} else if c == '?' {
 				l.r.Move(1)
 				return CommentToken, l.shiftBogusComment()
 			}
 		} else if 0 < len(l.tmplBegin) && l.at(l.tmplBegin...) {
+			if 0 < l.r.Pos() {
+				l.text = l.r.Shift()
+				return TextToken, l.text
+			}
 			l.r.Move(len(l.tmplBegin))
 			l.moveTemplate()
 			l.hasTmpl = true
+			return TemplateToken, l.r.Shift()
 		} else if c == 0 && l.r.Err() != nil {
 			if 0 < l.r.Pos() {
 				l.text = l.r.Shift()
