fix(context): count fullwidth chars in token estimates (#96442)

lin-hongkuan · web-flow · commit 31e941c3fc32 · 2026-06-25T14:36:04.000+08:00
diff --git a/src/utils/cjk-chars.test.ts b/src/utils/cjk-chars.test.ts
@@ -46,6 +46,13 @@ describe("estimateStringChars", () => {
     expect(estimateStringChars("안녕하세요")).toBe(20);
   });
 
+  it("handles East Asian fullwidth letters, numbers, and punctuation", () => {
+    expect(estimateStringChars("ＡＢＣ１２３")).toBe(6 * CHARS_PER_TOKEN_ESTIMATE);
+    expect(estimateStringChars("hello，world")).toBe(
+      "helloworld".length + CHARS_PER_TOKEN_ESTIMATE,
+    );
+  });
+
   it("handles CJK punctuation and symbols in the extended range", () => {
     // "⺀" (U+2E80) is in CJK Radicals Supplement range
     expect(estimateStringChars("⺀")).toBe(CHARS_PER_TOKEN_ESTIMATE);
diff --git a/src/utils/cjk-chars.ts b/src/utils/cjk-chars.ts
@@ -20,9 +20,10 @@ export const CHARS_PER_TOKEN_ESTIMATE = 4;
 /**
  * Matches CJK Unified Ideographs, CJK Extension A/B, CJK Compatibility
  * Ideographs, Hangul Syllables, Hiragana, Katakana, and other non-Latin
- * scripts that typically use ~1 token per character.
+ * scripts and East Asian fullwidth forms that typically use ~1 token per character.
  */
-const NON_LATIN_RE = /[\u2E80-\u9FFF\uA000-\uA4FF\uAC00-\uD7AF\uF900-\uFAFF\u{20000}-\u{2FA1F}]/gu;
+const NON_LATIN_RE =
+  /[\u2E80-\u9FFF\uA000-\uA4FF\uAC00-\uD7AF\uF900-\uFAFF\uFF01-\uFF60\uFFE0-\uFFE6\u{20000}-\u{2FA1F}]/gu;
 
 /**
  * Return an adjusted character length that accounts for non-Latin (CJK, etc.)
