@@ -88,7 +88,7 @@ export class Bm25Ranker implements Ranker {
 function tokenize(value: string): string[] {
 	return value
 		.toLowerCase()
-		.split(/[\W_]+/)
+		.split(/[^\p{L}\p{N}]+/u)
 		.filter(Boolean)
 }
 
 
@@ -59,6 +59,14 @@ describe("Bm25Ranker", () => {
 		expect(new Bm25Ranker().rank("create issue", [nonMatching, matching], 10)).toEqual([matching])
 	})
 
+	it("preserves Unicode letters and numbers in tokens", () => {
+		const russian = tool("погода-сервер", "прогноз", "Прогноз погоды")
+		const arabic = tool("أدوات", "مقياس", "يدعم ٣ بروتوكولات")
+
+		expect(new Bm25Ranker().rank("погода", [arabic, russian], 10)).toEqual([russian])
+		expect(new Bm25Ranker().rank("٣", [russian, arabic], 10)).toEqual([arabic])
+	})
+
 	it("handles tools with empty descriptions", () => {
 		const item = tool("GitHub", "listRepos", "")
Original file line number	Diff line number	Diff line change
`@@ -88,7 +88,7 @@ export class Bm25Ranker implements Ranker {`
`88`	`88`	`function tokenize(value: string): string[] {`
`89`	`89`	`return value`
`90`	`90`	`.toLowerCase()`
`91`		`- .split(/[\W_]+/)`
	`91`	`+ .split(/[^\p{L}\p{N}]+/u)`
`92`	`92`	`.filter(Boolean)`
`93`	`93`	`}`
`94`	`94`