refactor(zalouser): share offset-preserving text chunk ranges (#105842)

steipete · web-flow · commit 4cbbb86d5e71 · 2026-07-12T23:46:16.000-07:00
* refactor(zalouser): share text chunk ranges

* chore(plugin-sdk): refresh combined API baseline
diff --git a/docs/.generated/plugin-sdk-api-baseline.sha256 b/docs/.generated/plugin-sdk-api-baseline.sha256
@@ -1,2 +1,2 @@
-bc84c5da613ba34f84e9265ed0580ae99d8793a6033519dee33e11863daaff62  plugin-sdk-api-baseline.json
-a25e6c9ae393270029eb39de5e00a45f5a3ba13e526ff7532668494f3772159b  plugin-sdk-api-baseline.jsonl
+21f63180d1606d450af8d2509c2e12ae7bcb0e92b77429ee35b5d350d4f9a427  plugin-sdk-api-baseline.json
+7ea611a65949cbd82c54a63f3fc1818d88bf6b6515fcb53ef59a911b2ca4a822  plugin-sdk-api-baseline.jsonl
diff --git a/docs/plugins/sdk-migration.md b/docs/plugins/sdk-migration.md
@@ -500,7 +500,7 @@ SDK.
   | `plugin-sdk/media-generation-runtime` | Shared media-generation helpers | Shared failover helpers, candidate selection, and missing-model messaging for image/video/music generation |
   | `plugin-sdk/media-understanding` | Media-understanding helpers | Media understanding provider types plus provider-facing image/audio helper exports |
   | `plugin-sdk/text-runtime` | Deprecated broad text compatibility export | Use `string-coerce-runtime`, `text-chunking`, `text-utility-runtime`, and `logging-core` |
-  | `plugin-sdk/text-chunking` | Text chunking helpers | Outbound text chunking helper |
+  | `plugin-sdk/text-chunking` | Text chunking helpers | Outbound text and offset-preserving range chunking helpers |
   | `plugin-sdk/speech` | Speech helpers | Speech provider types plus provider-facing directive, registry, validation helpers, and OpenAI-compatible TTS builder |
   | `plugin-sdk/speech-core` | Shared speech core | Speech provider types, registry, directives, normalization |
   | `plugin-sdk/realtime-transcription` | Realtime transcription helpers | Provider types, registry helpers, and shared WebSocket session helper |
diff --git a/docs/plugins/sdk-subpaths.md b/docs/plugins/sdk-subpaths.md
@@ -352,7 +352,7 @@ usage endpoint failed or returned no usable usage data.
     | `plugin-sdk/media-store` | Narrow media store helpers such as `saveMediaBuffer` and `saveMediaStream` |
     | `plugin-sdk/media-generation-runtime` | Shared media-generation failover helpers, candidate selection, and missing-model messaging |
     | `plugin-sdk/media-understanding` | Media understanding provider types plus provider-facing image/audio/structured-extraction helper exports |
-    | `plugin-sdk/text-chunking` | Outbound text and markdown chunking/render helpers, markdown table conversion, directive-tag stripping, and safe-text utilities |
+    | `plugin-sdk/text-chunking` | Outbound text and offset-preserving range chunking, markdown chunking/render helpers, markdown table conversion, directive-tag stripping, and safe-text utilities |
     | `plugin-sdk/speech` | Speech provider types plus provider-facing directive, registry, validation, OpenAI-compatible TTS builder, and speech helper exports |
     | `plugin-sdk/speech-core` | Shared speech provider types, registry, directive, normalization, and speech helper exports |
     | `plugin-sdk/realtime-transcription` | Realtime transcription provider types, registry helpers, and shared WebSocket session helper |
diff --git a/extensions/zalouser/src/send.test.ts b/extensions/zalouser/src/send.test.ts
@@ -296,6 +296,42 @@ describe("zalouser send helpers", () => {
     expectResultFields(result, { ok: true, messageId: "mid-2d-4" });
   });
 
+  it.each([
+    {
+      name: "fenced code with hard limits",
+      text: "```ts\nconst alpha = 1;\nconst beta = 2;\n```",
+      textChunkLimit: 16,
+      textChunkMode: "length" as const,
+      expected: ["const alpha = 1;", "\nconst beta = 2;"],
+    },
+    {
+      name: "tables with newline-preferred limits",
+      text: "| A | B |\n| --- | --- |\n| 1 | 2 |\n| 3 | 4 |",
+      textChunkLimit: 24,
+      textChunkMode: "newline" as const,
+      expected: ["| A | B |\n| --- | --- |\n", "| 1 | 2 |\n| 3 | 4 |"],
+    },
+    {
+      name: "repeated whitespace around paragraph boundaries",
+      text: "alpha  beta\n\ngamma delta",
+      textChunkLimit: 12,
+      textChunkMode: "newline" as const,
+      expected: ["alpha  beta\n", "\ngamma delta"],
+    },
+  ])("preserves exact rendered chunks for $name", async (fixture) => {
+    mockSendText.mockResolvedValue(sendResult("mid-parity", "thread-parity"));
+
+    await sendMessageZalouser("thread-parity", fixture.text, {
+      textMode: "markdown",
+      textChunkLimit: fixture.textChunkLimit,
+      textChunkMode: fixture.textChunkMode,
+    });
+
+    const renderedChunks = mockSendText.mock.calls.map((call) => call[1]);
+    expect(renderedChunks).toEqual(fixture.expected);
+    expect(renderedChunks.join("")).toBe(parseZalouserTextStyles(fixture.text).text);
+  });
+
   it("respects an explicit text chunk limit when splitting formatted markdown", async () => {
     const text = `**${"a".repeat(1501)}**`;
     mockSendText
diff --git a/extensions/zalouser/src/send.ts b/extensions/zalouser/src/send.ts
@@ -1,4 +1,5 @@
 // Zalouser plugin module implements send behavior.
+import { chunkTextRanges } from "openclaw/plugin-sdk/text-chunking";
 import { createZalouserSendReceipt } from "./send-receipt.js";
 import { parseZalouserTextStyles } from "./text-styles.js";
 import type { ZaloEventMessage, ZaloSendOptions, ZaloSendResult } from "./types.js";
@@ -19,15 +20,12 @@ type ZalouserSendOptions = ZaloSendOptions & {
 type ZalouserSendResult = ZaloSendResult;
 
 const ZALO_TEXT_LIMIT = 2000;
-const DEFAULT_TEXT_CHUNK_MODE = "length";
 
 type StyledTextChunk = {
   text: string;
   styles?: ZaloSendOptions["textStyles"];
 };
 
-type TextChunkMode = NonNullable<ZaloSendOptions["textChunkMode"]>;
-
 export async function sendMessageZalouser(
   threadId: string,
   text: string,
@@ -155,7 +153,10 @@ function splitStyledText(
   }
 
   const chunks: StyledTextChunk[] = [];
-  for (const range of splitTextRanges(text, limit, mode ?? DEFAULT_TEXT_CHUNK_MODE)) {
+  for (const range of chunkTextRanges(text, {
+    limit,
+    mode: mode === "newline" ? "preferred" : "hard",
+  })) {
     const { start, end } = range;
     chunks.push({
       text: text.slice(start, end),
@@ -201,86 +202,3 @@ function sliceTextStyles(
 
   return chunkStyles.length > 0 ? chunkStyles : undefined;
 }
-
-function splitTextRanges(
-  text: string,
-  limit: number,
-  mode: TextChunkMode,
-): Array<{ start: number; end: number }> {
-  if (mode === "newline") {
-    return splitTextRangesByPreferredBreaks(text, limit);
-  }
-
-  const ranges: Array<{ start: number; end: number }> = [];
-  for (let start = 0; start < text.length; start += limit) {
-    ranges.push({
-      start,
-      end: Math.min(text.length, start + limit),
-    });
-  }
-  return ranges;
-}
-
-function splitTextRangesByPreferredBreaks(
-  text: string,
-  limit: number,
-): Array<{ start: number; end: number }> {
-  const ranges: Array<{ start: number; end: number }> = [];
-  let start = 0;
-
-  while (start < text.length) {
-    const maxEnd = Math.min(text.length, start + limit);
-    let end = maxEnd;
-    if (maxEnd < text.length) {
-      end =
-        findParagraphBreak(text, start, maxEnd) ??
-        findLastBreak(text, "\n", start, maxEnd) ??
-        findLastWhitespaceBreak(text, start, maxEnd) ??
-        maxEnd;
-    }
-
-    if (end <= start) {
-      end = maxEnd;
-    }
-
-    ranges.push({ start, end });
-    start = end;
-  }
-
-  return ranges;
-}
-
-function findParagraphBreak(text: string, start: number, end: number): number | undefined {
-  const slice = text.slice(start, end);
-  const matches = slice.matchAll(/\n[\t ]*\n+/g);
-  let lastMatch: RegExpMatchArray | undefined;
-  for (const match of matches) {
-    lastMatch = match;
-  }
-  if (!lastMatch || lastMatch.index === undefined) {
-    return undefined;
-  }
-  return start + lastMatch.index + lastMatch[0].length;
-}
-
-function findLastBreak(
-  text: string,
-  marker: string,
-  start: number,
-  end: number,
-): number | undefined {
-  const index = text.lastIndexOf(marker, end - 1);
-  if (index < start) {
-    return undefined;
-  }
-  return index + marker.length;
-}
-
-function findLastWhitespaceBreak(text: string, start: number, end: number): number | undefined {
-  for (let index = end - 1; index > start; index -= 1) {
-    if (/\s/.test(text.charAt(index))) {
-      return index + 1;
-    }
-  }
-  return undefined;
-}
diff --git a/packages/markdown-core/src/chunk-text.ts b/packages/markdown-core/src/chunk-text.ts
@@ -42,6 +42,69 @@ function scanParenAwareBreakpoints(text: string): { lastNewline: number; lastWhi
   return { lastNewline, lastWhitespace };
 }
 
+export type TextChunkRange = {
+  start: number;
+  end: number;
+};
+
+export type ChunkTextRangesOptions = {
+  limit: number;
+  mode?: "hard" | "preferred";
+};
+
+function findPreferredRangeEnd(text: string, start: number, end: number): number | undefined {
+  const slice = text.slice(start, end);
+  let paragraphEnd: number | undefined;
+  for (const match of slice.matchAll(/\n[\t ]*\n+/g)) {
+    if (match.index !== undefined) {
+      paragraphEnd = start + match.index + match[0].length;
+    }
+  }
+  if (paragraphEnd !== undefined) {
+    return paragraphEnd;
+  }
+
+  const newlineIndex = text.lastIndexOf("\n", end - 1);
+  if (newlineIndex >= start) {
+    return newlineIndex + 1;
+  }
+
+  for (let index = end - 1; index > start; index -= 1) {
+    if (/\s/.test(text.charAt(index))) {
+      return index + 1;
+    }
+  }
+  return undefined;
+}
+
+/**
+ * Splits text into contiguous UTF-16 ranges without dropping separator whitespace.
+ * Preferred mode selects paragraph, newline, then whitespace boundaries.
+ */
+export function chunkTextRanges(text: string, options: ChunkTextRangesOptions): TextChunkRange[] {
+  if (!text) {
+    return [];
+  }
+  if (options.limit <= 0 || text.length <= options.limit) {
+    return [{ start: 0, end: text.length }];
+  }
+
+  const ranges: TextChunkRange[] = [];
+  let start = 0;
+  while (start < text.length) {
+    const maxEnd = Math.min(text.length, start + options.limit);
+    const preferredEnd =
+      options.mode === "preferred" && maxEnd < text.length
+        ? findPreferredRangeEnd(text, start, maxEnd)
+        : undefined;
+    const candidateEnd = preferredEnd && preferredEnd > start ? preferredEnd : maxEnd;
+    const end = avoidTrailingHighSurrogateBreak(text, start, candidateEnd);
+    ranges.push({ start, end });
+    start = end;
+  }
+  return ranges;
+}
+
 /**
  * Keeps UTF-16 chunk boundaries from separating a supplementary-plane character.
  * A one-unit positive limit still needs to emit an entire surrogate pair.
diff --git a/packages/markdown-core/src/index.ts b/packages/markdown-core/src/index.ts
@@ -1,4 +1,5 @@
 /** Public Markdown parsing, rendering, chunking, and table-conversion utilities. */
+export * from "./chunk-text.js";
 export * from "./code-spans.js";
 export * from "./fences.js";
 export * from "./frontmatter.js";
diff --git a/scripts/plugin-sdk-surface-report.mjs b/scripts/plugin-sdk-surface-report.mjs
@@ -203,12 +203,12 @@ export function readPluginSdkSurfaceBudgets(env = process.env) {
     ),
     publicExports: readPluginSdkSurfaceBudgetEnv(
       "OPENCLAW_PLUGIN_SDK_MAX_PUBLIC_EXPORTS",
-      10641,
+      10644,
       env,
     ),
     publicFunctionExports: readPluginSdkSurfaceBudgetEnv(
       "OPENCLAW_PLUGIN_SDK_MAX_PUBLIC_FUNCTION_EXPORTS",
-      5357,
+      5358,
       env,
     ),
     publicDeprecatedExports: readPluginSdkSurfaceBudgetEnv(
diff --git a/src/plugin-sdk/text-chunking.test.ts b/src/plugin-sdk/text-chunking.test.ts
@@ -2,7 +2,7 @@
  * Tests text and Markdown chunking helpers exported by the plugin SDK.
  */
 import { describe, expect, it } from "vitest";
-import { chunkTextForOutbound } from "./text-chunking.js";
+import { chunkTextForOutbound, chunkTextRanges } from "./text-chunking.js";
 
 describe("chunkTextForOutbound", () => {
   it.each([
@@ -28,3 +28,44 @@ describe("chunkTextForOutbound", () => {
     expect(chunkTextForOutbound(text, maxLen)).toEqual(expected);
   });
 });
+
+describe("chunkTextRanges", () => {
+  it("returns contiguous hard ranges without dropping whitespace", () => {
+    const text = "alpha  beta\n\ngamma delta";
+    const ranges = chunkTextRanges(text, { limit: 12, mode: "hard" });
+
+    expect(ranges).toEqual([
+      { start: 0, end: 12 },
+      { start: 12, end: 24 },
+    ]);
+    expect(ranges.map(({ start, end }) => text.slice(start, end))).toEqual([
+      "alpha  beta\n",
+      "\ngamma delta",
+    ]);
+  });
+
+  it("prefers paragraph, newline, then whitespace boundaries", () => {
+    const text = "a\n\nb\nc xyz";
+    const ranges = chunkTextRanges(text, { limit: 8, mode: "preferred" });
+
+    expect(ranges.map(({ start, end }) => text.slice(start, end))).toEqual(["a\n\n", "b\nc xyz"]);
+  });
+
+  it("falls back to hard ranges and handles empty or non-positive limits", () => {
+    expect(chunkTextRanges("abcdefgh", { limit: 3, mode: "preferred" })).toEqual([
+      { start: 0, end: 3 },
+      { start: 3, end: 6 },
+      { start: 6, end: 8 },
+    ]);
+    expect(chunkTextRanges("", { limit: 3 })).toEqual([]);
+    expect(chunkTextRanges("abc", { limit: 0 })).toEqual([{ start: 0, end: 3 }]);
+  });
+
+  it.each(["hard", "preferred"] as const)("keeps surrogate pairs intact in %s mode", (mode) => {
+    expect(chunkTextRanges("a😀b", { limit: 2, mode })).toEqual([
+      { start: 0, end: 1 },
+      { start: 1, end: 3 },
+      { start: 3, end: 4 },
+    ]);
+  });
+});
diff --git a/src/plugin-sdk/text-chunking.ts b/src/plugin-sdk/text-chunking.ts
@@ -1,6 +1,13 @@
 // Text chunking helpers split long outbound text while preserving readable line boundaries.
 import { chunkTextByBreakResolver } from "../shared/text-chunking.js";
 
+/** Offset-preserving text ranges for transports with native style metadata. */
+export {
+  chunkTextRanges,
+  type ChunkTextRangesOptions,
+  type TextChunkRange,
+} from "../../packages/markdown-core/src/chunk-text.js";
+
 /**
  * Splits outbound channel text into chunks no longer than the requested limit.
  * Newline boundaries win over spaces; text without usable separators falls back

Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,5 @@`
`1`	`1`	`/** Public Markdown parsing, rendering, chunking, and table-conversion utilities. */`
	`2`	`+export * from "./chunk-text.js";`
`2`	`3`	`export * from "./code-spans.js";`
`3`	`4`	`export * from "./fences.js";`
`4`	`5`	`export * from "./frontmatter.js";`