Skip to content

Commit 56d1869

Browse files
committed
refactor(extraction): decode documents from required buffers
Extract the document-decoding boundary from PR #1999 as an independent prerequisite on main. Keep explicit path-only adapters, require supplied bytes for PDF/DOCX/IPYNB/XLSX decoders, expose a disk-free buffer API, and separate workbook formatting. Preserve historical formatted output and cover ordinary/empty/binary buffers, notebook formatting, workbook bytes, and path-adapter compatibility without reader safety changes.
1 parent ecab519 commit 56d1869

4 files changed

Lines changed: 192 additions & 50 deletions

File tree

Lines changed: 33 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,10 @@
11
import ExcelJS from "exceljs"
2-
import { extractTextFromXLSX } from "../extract-text-from-xlsx"
2+
import { formatWorkbook } from "../extract-text-from-xlsx"
3+
import { extractTextFromFile } from "../extract-text"
34

4-
describe("extractTextFromXLSX", () => {
5+
describe("formatWorkbook", () => {
56
describe("basic functionality", () => {
6-
it("should extract text with proper formatting", async () => {
7+
it("should extract text with proper formatting", () => {
78
const workbook = new ExcelJS.Workbook()
89
const worksheet = workbook.addWorksheet("Sheet1")
910

@@ -12,22 +13,22 @@ describe("extractTextFromXLSX", () => {
1213
worksheet.getCell("A2").value = "Test"
1314
worksheet.getCell("B2").value = 123
1415

15-
const result = await extractTextFromXLSX(workbook)
16+
const result = formatWorkbook(workbook)
1617

1718
expect(result).toContain("--- Sheet: Sheet1 ---")
1819
expect(result).toContain("Hello\tWorld")
1920
expect(result).toContain("Test\t123")
2021
})
2122

22-
it("should skip rows with no content", async () => {
23+
it("should skip rows with no content", () => {
2324
const workbook = new ExcelJS.Workbook()
2425
const worksheet = workbook.addWorksheet("Sheet1")
2526

2627
worksheet.getCell("A1").value = "Row 1"
2728
// Row 2 is completely empty
2829
worksheet.getCell("A3").value = "Row 3"
2930

30-
const result = await extractTextFromXLSX(workbook)
31+
const result = formatWorkbook(workbook)
3132

3233
expect(result).toContain("Row 1")
3334
expect(result).toContain("Row 3")
@@ -37,7 +38,7 @@ describe("extractTextFromXLSX", () => {
3738
})
3839

3940
describe("sheet handling", () => {
40-
it("should process multiple sheets", async () => {
41+
it("should process multiple sheets", () => {
4142
const workbook = new ExcelJS.Workbook()
4243

4344
const sheet1 = workbook.addWorksheet("First Sheet")
@@ -46,15 +47,15 @@ describe("extractTextFromXLSX", () => {
4647
const sheet2 = workbook.addWorksheet("Second Sheet")
4748
sheet2.getCell("A1").value = "Sheet 2 Data"
4849

49-
const result = await extractTextFromXLSX(workbook)
50+
const result = formatWorkbook(workbook)
5051

5152
expect(result).toContain("--- Sheet: First Sheet ---")
5253
expect(result).toContain("Sheet 1 Data")
5354
expect(result).toContain("--- Sheet: Second Sheet ---")
5455
expect(result).toContain("Sheet 2 Data")
5556
})
5657

57-
it("should skip hidden sheets", async () => {
58+
it("should skip hidden sheets", () => {
5859
const workbook = new ExcelJS.Workbook()
5960

6061
const visibleSheet = workbook.addWorksheet("Visible Sheet")
@@ -64,15 +65,15 @@ describe("extractTextFromXLSX", () => {
6465
hiddenSheet.getCell("A1").value = "Hidden Data"
6566
hiddenSheet.state = "hidden"
6667

67-
const result = await extractTextFromXLSX(workbook)
68+
const result = formatWorkbook(workbook)
6869

6970
expect(result).toContain("--- Sheet: Visible Sheet ---")
7071
expect(result).toContain("Visible Data")
7172
expect(result).not.toContain("--- Sheet: Hidden Sheet ---")
7273
expect(result).not.toContain("Hidden Data")
7374
})
7475

75-
it("should skip very hidden sheets", async () => {
76+
it("should skip very hidden sheets", () => {
7677
const workbook = new ExcelJS.Workbook()
7778

7879
const visibleSheet = workbook.addWorksheet("Visible Sheet")
@@ -82,7 +83,7 @@ describe("extractTextFromXLSX", () => {
8283
veryHiddenSheet.getCell("A1").value = "Very Hidden Data"
8384
veryHiddenSheet.state = "veryHidden"
8485

85-
const result = await extractTextFromXLSX(workbook)
86+
const result = formatWorkbook(workbook)
8687

8788
expect(result).toContain("--- Sheet: Visible Sheet ---")
8889
expect(result).toContain("Visible Data")
@@ -92,7 +93,7 @@ describe("extractTextFromXLSX", () => {
9293
})
9394

9495
describe("formatCellValue logic", () => {
95-
it("should handle null and undefined values", async () => {
96+
it("should handle null and undefined values", () => {
9697
const workbook = new ExcelJS.Workbook()
9798
const worksheet = workbook.addWorksheet("Sheet1")
9899

@@ -101,7 +102,7 @@ describe("extractTextFromXLSX", () => {
101102
worksheet.getCell("A3").value = undefined
102103
worksheet.getCell("A4").value = "After"
103104

104-
const result = await extractTextFromXLSX(workbook)
105+
const result = formatWorkbook(workbook)
105106

106107
expect(result).toContain("Before")
107108
expect(result).toContain("After")
@@ -111,43 +112,43 @@ describe("extractTextFromXLSX", () => {
111112
expect(dataLines).toHaveLength(2) // Only 'Before' and 'After' should create content
112113
})
113114

114-
it("should format dates correctly", async () => {
115+
it("should format dates correctly", () => {
115116
const workbook = new ExcelJS.Workbook()
116117
const worksheet = workbook.addWorksheet("Sheet1")
117118

118119
const testDate = new Date("2023-12-25")
119120
worksheet.getCell("A1").value = testDate
120121

121-
const result = await extractTextFromXLSX(workbook)
122+
const result = formatWorkbook(workbook)
122123

123124
expect(result).toContain("2023-12-25")
124125
})
125126

126-
it("should handle error values", async () => {
127+
it("should handle error values", () => {
127128
const workbook = new ExcelJS.Workbook()
128129
const worksheet = workbook.addWorksheet("Sheet1")
129130

130131
worksheet.getCell("A1").value = { error: "#DIV/0!" }
131132

132-
const result = await extractTextFromXLSX(workbook)
133+
const result = formatWorkbook(workbook)
133134

134135
expect(result).toContain("[Error: #DIV/0!]")
135136
})
136137

137-
it("should handle rich text", async () => {
138+
it("should handle rich text", () => {
138139
const workbook = new ExcelJS.Workbook()
139140
const worksheet = workbook.addWorksheet("Sheet1")
140141

141142
worksheet.getCell("A1").value = {
142143
richText: [{ text: "Hello " }, { text: "World", font: { bold: true } }],
143144
}
144145

145-
const result = await extractTextFromXLSX(workbook)
146+
const result = formatWorkbook(workbook)
146147

147148
expect(result).toContain("Hello World")
148149
})
149150

150-
it("should handle hyperlinks", async () => {
151+
it("should handle hyperlinks", () => {
151152
const workbook = new ExcelJS.Workbook()
152153
const worksheet = workbook.addWorksheet("Sheet1")
153154

@@ -156,45 +157,45 @@ describe("extractTextFromXLSX", () => {
156157
hyperlink: "https://roocode.com/",
157158
}
158159

159-
const result = await extractTextFromXLSX(workbook)
160+
const result = formatWorkbook(workbook)
160161

161162
expect(result).toContain("Roo Code (https://roocode.com/)")
162163
})
163164

164-
it("should handle formulas with and without results", async () => {
165+
it("should handle formulas with and without results", () => {
165166
const workbook = new ExcelJS.Workbook()
166167
const worksheet = workbook.addWorksheet("Sheet1")
167168

168169
worksheet.getCell("A1").value = { formula: "A2+A3", result: 30 }
169170
worksheet.getCell("A2").value = { formula: "SUM(B1:B10)" }
170171

171-
const result = await extractTextFromXLSX(workbook)
172+
const result = formatWorkbook(workbook)
172173

173174
expect(result).toContain("30") // Formula with result
174175
expect(result).toContain("[Formula: SUM(B1:B10)]") // Formula without result
175176
})
176177
})
177178

178179
describe("edge cases", () => {
179-
it("should handle empty workbook", async () => {
180+
it("should handle empty workbook", () => {
180181
const workbook = new ExcelJS.Workbook()
181182
workbook.addWorksheet("Empty Sheet")
182183

183-
const result = await extractTextFromXLSX(workbook)
184+
const result = formatWorkbook(workbook)
184185

185186
expect(result).toContain("--- Sheet: Empty Sheet ---")
186187
expect(result.trim()).toBe("--- Sheet: Empty Sheet ---")
187188
})
188189

189-
it("should handle workbook with only empty cells", async () => {
190+
it("should handle workbook with only empty cells", () => {
190191
const workbook = new ExcelJS.Workbook()
191192
const worksheet = workbook.addWorksheet("Sheet1")
192193

193194
// Set cells but leave them empty
194195
worksheet.getCell("A1").value = ""
195196
worksheet.getCell("B1").value = ""
196197

197-
const result = await extractTextFromXLSX(workbook)
198+
const result = formatWorkbook(workbook)
198199

199200
expect(result).toContain("--- Sheet: Sheet1 ---")
200201
// Should not contain any data rows since empty strings don't count as content
@@ -203,19 +204,19 @@ describe("extractTextFromXLSX", () => {
203204
})
204205
})
205206

206-
describe("function overloads", () => {
207-
it("should work with workbook objects", async () => {
207+
describe("workbook formatting and path adapter", () => {
208+
it("should work with workbook objects", () => {
208209
const workbook = new ExcelJS.Workbook()
209210
const worksheet = workbook.addWorksheet("Test")
210211
worksheet.getCell("A1").value = "Test Data"
211212

212-
const result = await extractTextFromXLSX(workbook)
213+
const result = formatWorkbook(workbook)
213214

214215
expect(result).toContain("Test Data")
215216
})
216217

217218
it("should reject invalid file paths", async () => {
218-
await expect(extractTextFromXLSX("/non/existent/file.xlsx")).rejects.toThrow()
219+
await expect(extractTextFromFile("/non/existent/file.xlsx")).rejects.toThrow()
219220
})
220221
})
221222
})

‎src/integrations/misc/__tests__/extract-text.spec.ts‎

Lines changed: 125 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,10 @@
1+
import fs from "fs/promises"
2+
import * as binaryFile from "isbinaryfile"
3+
import ExcelJS from "exceljs"
14
import {
25
addLineNumbers,
6+
extractTextFromBuffer,
7+
extractTextFromFileWithMetadata,
38
everyLineHasLineNumbers,
49
stripLineNumbers,
510
truncateOutput,
@@ -8,6 +13,126 @@ import {
813
processBackspaces,
914
} from "../extract-text"
1015

16+
describe("buffer input", () => {
17+
afterEach(() => vi.restoreAllMocks())
18+
19+
it("returns supplied ordinary-text bytes without pathname I/O", async () => {
20+
const access = vi.spyOn(fs, "access").mockResolvedValue(undefined)
21+
const readFile = vi.spyOn(fs, "readFile").mockResolvedValue("unapproved disk content")
22+
const probe = vi.spyOn(binaryFile, "isBinaryFile").mockResolvedValue(false)
23+
const source = Buffer.from("approved first line\napproved second line")
24+
25+
const content = await extractTextFromBuffer(source, "metadata-only.txt")
26+
27+
expect(content).toBe("1 | approved first line\n2 | approved second line")
28+
expect(access).not.toHaveBeenCalled()
29+
expect(readFile).not.toHaveBeenCalled()
30+
expect(probe).not.toHaveBeenCalledWith("metadata-only.txt")
31+
})
32+
33+
it("preserves empty-text formatting for an empty supplied buffer without pathname I/O", async () => {
34+
const access = vi.spyOn(fs, "access").mockRejectedValue(new Error("Unexpected pathname access"))
35+
const readFile = vi.spyOn(fs, "readFile").mockRejectedValue(new Error("Unexpected pathname read"))
36+
const probe = vi.spyOn(binaryFile, "isBinaryFile")
37+
const source = Buffer.alloc(0)
38+
39+
const content = await extractTextFromBuffer(source, "missing.txt")
40+
41+
expect(content).toBe("1 | ")
42+
expect(access).not.toHaveBeenCalled()
43+
expect(readFile).not.toHaveBeenCalled()
44+
expect(probe).not.toHaveBeenCalledWith("missing.txt")
45+
})
46+
47+
it("rejects unsupported binary bytes without pathname I/O", async () => {
48+
const access = vi.spyOn(fs, "access").mockRejectedValue(new Error("Unexpected pathname access"))
49+
const readFile = vi.spyOn(fs, "readFile").mockRejectedValue(new Error("Unexpected pathname read"))
50+
const probe = vi.spyOn(binaryFile, "isBinaryFile")
51+
const source = Buffer.from([0, 1, 2, 0, 255])
52+
53+
await expect(extractTextFromBuffer(source, "missing.BIN")).rejects.toThrow(
54+
"Cannot read text for file type: .bin",
55+
)
56+
57+
expect(access).not.toHaveBeenCalled()
58+
expect(readFile).not.toHaveBeenCalled()
59+
expect(probe).not.toHaveBeenCalledWith("missing.BIN")
60+
})
61+
62+
it("decodes notebook bytes with historical line numbering and trailing newline", async () => {
63+
const access = vi.spyOn(fs, "access").mockRejectedValue(new Error("Unexpected pathname access"))
64+
const readFile = vi.spyOn(fs, "readFile").mockRejectedValue(new Error("Unexpected pathname read"))
65+
const source = Buffer.from(
66+
JSON.stringify({
67+
cells: [
68+
{ cell_type: "markdown", source: ["Heading"] },
69+
{ cell_type: "code", source: ["first", "second"] },
70+
{ cell_type: "raw", source: ["ignored"] },
71+
],
72+
}),
73+
)
74+
75+
expect(await extractTextFromBuffer(source, "missing.IPYNB")).toBe("1 | Heading\n2 | first\n3 | second\n")
76+
expect(access).not.toHaveBeenCalled()
77+
expect(readFile).not.toHaveBeenCalled()
78+
})
79+
80+
it("preserves empty notebook output without pathname I/O", async () => {
81+
const access = vi.spyOn(fs, "access").mockRejectedValue(new Error("Unexpected pathname access"))
82+
const readFile = vi.spyOn(fs, "readFile").mockRejectedValue(new Error("Unexpected pathname read"))
83+
const source = Buffer.from(JSON.stringify({ cells: [] }))
84+
85+
expect(await extractTextFromBuffer(source, "missing.ipynb")).toBe("")
86+
expect(access).not.toHaveBeenCalled()
87+
expect(readFile).not.toHaveBeenCalled()
88+
})
89+
90+
it("decodes XLSX bytes and preserves visible-sheet formatting without pathname I/O", async () => {
91+
const workbook = new ExcelJS.Workbook()
92+
workbook.addWorksheet("Data").addRow(["approved cell", 42])
93+
const hidden = workbook.addWorksheet("Hidden", { state: "hidden" })
94+
hidden.addRow(["hidden cell"])
95+
const source = Buffer.from(await workbook.xlsx.writeBuffer())
96+
const access = vi.spyOn(fs, "access").mockRejectedValue(new Error("Unexpected pathname access"))
97+
const readFile = vi.spyOn(fs, "readFile").mockRejectedValue(new Error("Unexpected pathname read"))
98+
99+
expect(await extractTextFromBuffer(source, "missing.XLSX")).toBe("--- Sheet: Data ---\napproved cell\t42")
100+
expect(access).not.toHaveBeenCalled()
101+
expect(readFile).not.toHaveBeenCalled()
102+
})
103+
})
104+
105+
describe("path input", () => {
106+
afterEach(() => vi.restoreAllMocks())
107+
108+
it("rejects unsupported binary paths after probing without loading the whole file", async () => {
109+
vi.spyOn(fs, "access").mockResolvedValue(undefined)
110+
const readFile = vi.spyOn(fs, "readFile").mockRejectedValue(new Error("Unexpected whole-file read"))
111+
const probe = vi.spyOn(binaryFile, "isBinaryFile").mockResolvedValue(true)
112+
113+
await expect(extractTextFromFileWithMetadata("large.bin")).rejects.toThrow(
114+
"Cannot read text for file type: .bin",
115+
)
116+
117+
expect(probe).toHaveBeenCalledWith("large.bin")
118+
expect(readFile).not.toHaveBeenCalled()
119+
})
120+
121+
it("loads document bytes from the path and preserves document metadata", async () => {
122+
vi.spyOn(fs, "access").mockResolvedValue(undefined)
123+
const source = Buffer.from(JSON.stringify({ cells: [{ cell_type: "code", source: ["first", "second"] }] }))
124+
const readFile = vi.spyOn(fs, "readFile").mockResolvedValue(source)
125+
126+
expect(await extractTextFromFileWithMetadata("document.ipynb", 1)).toEqual({
127+
content: "1 | first\n2 | second\n",
128+
totalLines: 3,
129+
returnedLines: 3,
130+
wasTruncated: false,
131+
})
132+
expect(readFile).toHaveBeenCalledWith("document.ipynb")
133+
})
134+
})
135+
11136
describe("addLineNumbers", () => {
12137
it("should add line numbers starting from 1 by default", () => {
13138
const input = "line 1\nline 2\nline 3"

0 commit comments

Comments
 (0)