revert: splitIntoPotentialTokens lookup tables

claude · claude · commit bc7d55e99c9c · 2026-04-22T18:06:35.000Z
CodSpeed reports a ~19% instruction-count regression on both `helpers/splitIntoPotentialTokens` benchmarks (and ~11% on the `original-source` streamChunks benchmarks that call it) when the chained `===` comparisons are replaced with Uint8Array lookups. Restore the comparison-chain form; the other perf changes from the previous commit (readMappings, streamChunksOfSourceMap, createMappingsSerializer, ConcatSource, streamChunksOfCombinedSourceMap) are unaffected. https://claude.ai/code/session_013RELTj96iEXrmMSPxnwjeR
diff --git a/lib/helpers/splitIntoPotentialTokens.js b/lib/helpers/splitIntoPotentialTokens.js
@@ -13,25 +13,6 @@
 // \r = 13
 // \t = 9
 
-// Two Uint8Array lookup tables replace the chained `===` comparisons in the
-// hot scan loops. V8 keeps the tables in L1 as a constant, so the inner
-// condition becomes a single bounds check plus a typed-array load, which is
-// cheaper than 4–6 branches per character for long inputs.
-// Indexed by charCode; entries outside the ASCII range are implicitly 0.
-const BOUNDARY = new Uint8Array(126);
-BOUNDARY[10] = 1; // \n
-BOUNDARY[59] = 1; // ;
-BOUNDARY[123] = 1; // {
-BOUNDARY[125] = 1; // }
-
-const SEPARATOR = new Uint8Array(126);
-SEPARATOR[9] = 1; // \t
-SEPARATOR[13] = 1; // \r
-SEPARATOR[32] = 1; // space
-SEPARATOR[59] = 1; // ;
-SEPARATOR[123] = 1; // {
-SEPARATOR[125] = 1; // }
-
 /**
  * @param {string} str string
  * @returns {string[] | null} array of string separated by potential tokens
@@ -45,14 +26,18 @@ const splitIntoPotentialTokens = (str) => {
 		const start = i;
 		block: {
 			let cc = str.charCodeAt(i);
-			// Advance through non-boundary characters. Non-ASCII codepoints
-			// (cc >= 126) are by definition not boundaries.
-			while (cc >= 126 || BOUNDARY[cc] === 0) {
+			while (cc !== 10 && cc !== 59 && cc !== 123 && cc !== 125) {
 				if (++i >= len) break block;
 				cc = str.charCodeAt(i);
 			}
-			// Consume trailing separators so they stay grouped with the token.
-			while (cc < 126 && SEPARATOR[cc] === 1) {
+			while (
+				cc === 59 ||
+				cc === 32 ||
+				cc === 123 ||
+				cc === 125 ||
+				cc === 13 ||
+				cc === 9
+			) {
 				if (++i >= len) break block;
 				cc = str.charCodeAt(i);
 			}
