Skip to content

Commit 5621f29

Browse files
charset: stop checking for ascii if utf8 is satisfied (#768)
Since ascii is a subset of utf8, we don't need to iterate over input bytes twice. Just checking once that the content if valid utf8 is enough. FromPlain-4 145.15n ± 2% 36.52n ± 6% -74.84% (p=0.002 n=6)
1 parent eb8d3f4 commit 5621f29

1 file changed

Lines changed: 1 addition & 21 deletions

File tree

‎internal/charset/charset.go‎

Lines changed: 1 addition & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -84,19 +84,8 @@ func FromPlain(content []byte) string {
8484
break
8585
}
8686
}
87-
hasHighBit := false
88-
for _, c := range content {
89-
if c >= 0x80 {
90-
hasHighBit = true
91-
break
92-
}
93-
}
94-
if hasHighBit && utf8.Valid(content) {
95-
return "utf-8"
96-
}
97-
9887
// ASCII is a subset of UTF8. Follow W3C recommendation and replace with UTF8.
99-
if ascii(origContent) {
88+
if utf8.Valid(content) {
10089
return "utf-8"
10190
}
10291

@@ -123,15 +112,6 @@ func latin(content []byte) string {
123112
return "iso-8859-1"
124113
}
125114

126-
func ascii(content []byte) bool {
127-
for _, b := range content {
128-
if textChars[b] != T {
129-
return false
130-
}
131-
}
132-
return true
133-
}
134-
135115
// FromXML returns the charset of an XML document. It relies on the XML
136116
// header <?xml version="1.0" encoding="UTF-8"?> and falls back on the plain
137117
// text content.

0 commit comments

Comments
 (0)