Skip to content

Commit 99ffd71

Browse files
steipetesteipete
authored
refactor: extract markdown core package (#88265)
* refactor: extract markdown core package * refactor: remove old markdown sources * fix: use source paths for markdown core imports * fix: clean markdown package dependency ownership * fix: refresh root shrinkwrap for markdown dependency move
1 parent 0f8ea1d commit 99ffd71

61 files changed

Lines changed: 455 additions & 114 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎config/knip.config.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -180,6 +180,10 @@ const config = {
180180
entry: ["src/index.ts!", "src/ip.ts!"],
181181
project: ["src/**/*.ts!"],
182182
},
183+
"packages/markdown-core": {
184+
entry: ["src/*.ts!"],
185+
project: ["src/**/*.ts!"],
186+
},
183187
"packages/speech-core": {
184188
entry: ["api.ts!", "runtime-api.ts!", "speaker.ts!", "voice-models.ts!"],
185189
project: ["**/*.ts!"],

‎extensions/tsconfig.package-boundary.paths.json‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -102,6 +102,36 @@
102102
"../dist/plugin-sdk/packages/llm-core/src/validation.d.ts"
103103
],
104104
"@openclaw/llm-core/*": ["../dist/plugin-sdk/packages/llm-core/src/*.d.ts"],
105+
"@openclaw/markdown-core": [
106+
"../dist/plugin-sdk/packages/markdown-core/src/index.d.ts"
107+
],
108+
"@openclaw/markdown-core/code-spans": [
109+
"../dist/plugin-sdk/packages/markdown-core/src/code-spans.d.ts"
110+
],
111+
"@openclaw/markdown-core/fences": [
112+
"../dist/plugin-sdk/packages/markdown-core/src/fences.d.ts"
113+
],
114+
"@openclaw/markdown-core/frontmatter": [
115+
"../dist/plugin-sdk/packages/markdown-core/src/frontmatter.d.ts"
116+
],
117+
"@openclaw/markdown-core/ir": [
118+
"../dist/plugin-sdk/packages/markdown-core/src/ir.d.ts"
119+
],
120+
"@openclaw/markdown-core/render": [
121+
"../dist/plugin-sdk/packages/markdown-core/src/render.d.ts"
122+
],
123+
"@openclaw/markdown-core/render-aware-chunking": [
124+
"../dist/plugin-sdk/packages/markdown-core/src/render-aware-chunking.d.ts"
125+
],
126+
"@openclaw/markdown-core/tables": [
127+
"../dist/plugin-sdk/packages/markdown-core/src/tables.d.ts"
128+
],
129+
"@openclaw/markdown-core/types": [
130+
"../dist/plugin-sdk/packages/markdown-core/src/types.d.ts"
131+
],
132+
"@openclaw/markdown-core/*": [
133+
"../dist/plugin-sdk/packages/markdown-core/src/*.d.ts"
134+
],
105135
"@openclaw/media-generation-core": [
106136
"../dist/plugin-sdk/packages/media-generation-core/src/index.d.ts"
107137
],

‎extensions/xai/tsconfig.json‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,36 @@
111111
"@openclaw/llm-core/*": [
112112
"../../dist/plugin-sdk/packages/llm-core/src/*.d.ts"
113113
],
114+
"@openclaw/markdown-core": [
115+
"../../dist/plugin-sdk/packages/markdown-core/src/index.d.ts"
116+
],
117+
"@openclaw/markdown-core/code-spans": [
118+
"../../dist/plugin-sdk/packages/markdown-core/src/code-spans.d.ts"
119+
],
120+
"@openclaw/markdown-core/fences": [
121+
"../../dist/plugin-sdk/packages/markdown-core/src/fences.d.ts"
122+
],
123+
"@openclaw/markdown-core/frontmatter": [
124+
"../../dist/plugin-sdk/packages/markdown-core/src/frontmatter.d.ts"
125+
],
126+
"@openclaw/markdown-core/ir": [
127+
"../../dist/plugin-sdk/packages/markdown-core/src/ir.d.ts"
128+
],
129+
"@openclaw/markdown-core/render": [
130+
"../../dist/plugin-sdk/packages/markdown-core/src/render.d.ts"
131+
],
132+
"@openclaw/markdown-core/render-aware-chunking": [
133+
"../../dist/plugin-sdk/packages/markdown-core/src/render-aware-chunking.d.ts"
134+
],
135+
"@openclaw/markdown-core/tables": [
136+
"../../dist/plugin-sdk/packages/markdown-core/src/tables.d.ts"
137+
],
138+
"@openclaw/markdown-core/types": [
139+
"../../dist/plugin-sdk/packages/markdown-core/src/types.d.ts"
140+
],
141+
"@openclaw/markdown-core/*": [
142+
"../../dist/plugin-sdk/packages/markdown-core/src/*.d.ts"
143+
],
114144
"@openclaw/media-generation-core": [
115145
"../../dist/plugin-sdk/packages/media-generation-core/src/index.d.ts"
116146
],

‎npm-shrinkwrap.json‎

Lines changed: 0 additions & 74 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎package.json‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1882,7 +1882,6 @@
18821882
"jszip": "3.10.1",
18831883
"kysely": "0.29.2",
18841884
"linkedom": "0.18.12",
1885-
"markdown-it": "14.2.0",
18861885
"minimatch": "10.2.5",
18871886
"node-edge-tts": "1.2.10",
18881887
"openai": "6.39.0",

‎packages/markdown-core/package.json‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
{
2+
"name": "@openclaw/markdown-core",
3+
"version": "0.0.0-private",
4+
"private": true,
5+
"files": [
6+
"dist"
7+
],
8+
"type": "module",
9+
"main": "./dist/index.mjs",
10+
"types": "./dist/index.d.mts",
11+
"exports": {
12+
".": {
13+
"types": "./dist/index.d.mts",
14+
"import": "./dist/index.mjs",
15+
"default": "./dist/index.mjs"
16+
},
17+
"./code-spans": {
18+
"types": "./dist/code-spans.d.mts",
19+
"import": "./dist/code-spans.mjs",
20+
"default": "./dist/code-spans.mjs"
21+
},
22+
"./fences": {
23+
"types": "./dist/fences.d.mts",
24+
"import": "./dist/fences.mjs",
25+
"default": "./dist/fences.mjs"
26+
},
27+
"./frontmatter": {
28+
"types": "./dist/frontmatter.d.mts",
29+
"import": "./dist/frontmatter.mjs",
30+
"default": "./dist/frontmatter.mjs"
31+
},
32+
"./ir": {
33+
"types": "./dist/ir.d.mts",
34+
"import": "./dist/ir.mjs",
35+
"default": "./dist/ir.mjs"
36+
},
37+
"./render": {
38+
"types": "./dist/render.d.mts",
39+
"import": "./dist/render.mjs",
40+
"default": "./dist/render.mjs"
41+
},
42+
"./render-aware-chunking": {
43+
"types": "./dist/render-aware-chunking.d.mts",
44+
"import": "./dist/render-aware-chunking.mjs",
45+
"default": "./dist/render-aware-chunking.mjs"
46+
},
47+
"./tables": {
48+
"types": "./dist/tables.d.mts",
49+
"import": "./dist/tables.mjs",
50+
"default": "./dist/tables.mjs"
51+
},
52+
"./types": {
53+
"types": "./dist/types.d.mts",
54+
"import": "./dist/types.mjs",
55+
"default": "./dist/types.mjs"
56+
}
57+
},
58+
"dependencies": {
59+
"markdown-it": "14.2.0",
60+
"yaml": "2.9.0"
61+
}
62+
}

‎packages/markdown-core/src/chunk-text.ts‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
function resolveChunkEarlyReturn(text: string, limit: number): string[] | undefined {
2+
if (!text) {
3+
return [];
4+
}
5+
if (limit <= 0) {
6+
return [text];
7+
}
8+
if (text.length <= limit) {
9+
return [text];
10+
}
11+
return undefined;
12+
}
13+
14+
function scanParenAwareBreakpoints(text: string): { lastNewline: number; lastWhitespace: number } {
15+
let lastNewline = -1;
16+
let lastWhitespace = -1;
17+
let depth = 0;
18+
19+
for (let i = 0; i < text.length; i++) {
20+
const char = text[i];
21+
if (char === "(") {
22+
depth += 1;
23+
continue;
24+
}
25+
if (char === ")" && depth > 0) {
26+
depth -= 1;
27+
continue;
28+
}
29+
if (depth !== 0) {
30+
continue;
31+
}
32+
if (char === "\n") {
33+
lastNewline = i;
34+
} else if (/\s/.test(char)) {
35+
lastWhitespace = i;
36+
}
37+
}
38+
39+
return { lastNewline, lastWhitespace };
40+
}
41+
42+
export function chunkText(text: string, limit: number): string[] {
43+
const early = resolveChunkEarlyReturn(text, limit);
44+
if (early) {
45+
return early;
46+
}
47+
48+
const chunks: string[] = [];
49+
let cursor = 0;
50+
while (cursor < text.length) {
51+
if (text.length - cursor <= limit) {
52+
chunks.push(text.slice(cursor));
53+
break;
54+
}
55+
const windowEnd = Math.min(text.length, cursor + limit);
56+
const window = text.slice(cursor, windowEnd);
57+
const { lastNewline, lastWhitespace } = scanParenAwareBreakpoints(window);
58+
const breakOffset = lastNewline > 0 ? lastNewline : lastWhitespace;
59+
const end = breakOffset > 0 ? cursor + breakOffset : windowEnd;
60+
chunks.push(text.slice(cursor, end));
61+
cursor = end;
62+
while (cursor < text.length && /\s/.test(text[cursor] ?? "")) {
63+
cursor += 1;
64+
}
65+
}
66+
return chunks;
67+
}

src/markdown/code-spans.ts renamed to packages/markdown-core/src/code-spans.ts

File renamed without changes.

src/markdown/fences.ts renamed to packages/markdown-core/src/fences.ts

File renamed without changes.

src/markdown/frontmatter.test.ts renamed to packages/markdown-core/src/frontmatter.test.ts

File renamed without changes.

0 commit comments

Comments
 (0)