Skip to content

Commit ae8199e

Browse files
WebMadWebMad
committed
test(extraction): make document fixtures reproducible
1 parent cddb4de commit ae8199e

3 files changed

Lines changed: 142 additions & 0 deletions

File tree

‎src/integrations/misc/__tests__/extract-text-formats.integration.test.ts‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,26 @@
11
import fs from "fs/promises"
22
import path from "path"
3+
import { createHash } from "node:crypto"
34
import { extractTextFromBuffer } from "../extract-text"
45

56
// Keep real parser fixtures separate from the mocked parser unit suite.
67
describe("real document buffer decoding", () => {
78
afterEach(() => vi.restoreAllMocks())
89

10+
it.each([
11+
{
12+
filename: "approved.pdf",
13+
sha256: "11b19aed8bc2e40b9eda9d18b7bfe36e0bbc41c4383d7e30141927992790f411",
14+
},
15+
{
16+
filename: "approved.docx",
17+
sha256: "732d827ad38ff51c24983c773bcb964ba53ca29bb055ea73aa9fbed820ae3a9b",
18+
},
19+
])("matches the documented SHA-256 for $filename", async ({ filename, sha256 }) => {
20+
const source = await fs.readFile(path.join(__dirname, "fixtures", filename))
21+
expect(createHash("sha256").update(source).digest("hex")).toBe(sha256)
22+
})
23+
924
it("extracts numbered PDF text from fixture bytes without reopening the metadata pathname", async () => {
1025
const source = await fs.readFile(path.join(__dirname, "fixtures", "approved.pdf"))
1126
const access = vi.spyOn(fs, "access").mockRejectedValue(new Error("Unexpected pathname access"))

‎src/integrations/misc/__tests__/fixtures/FIXTURES.md‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,3 +6,22 @@ These small, owned documents exercise the real parsers through the buffer-only e
66
- [approved.docx](approved.docx) is a minimal OOXML document generated using Python's standard-library ZIP writer, fixed entry timestamps, and DEFLATE compression. It includes the content-type declarations, root office-document relationship, and main document part. Its two paragraphs read “Approved DOCX first paragraph” and “Approved DOCX second & final paragraph”; the second combines two runs and an escaped XML entity. Mammoth separates paragraphs with two newlines.
77

88
The integration suite deliberately does not mock either document parser. It reads each fixture before guarding against pathname I/O, then passes a different, metadata-only pathname to the extraction API and asserts the full numbered output.
9+
10+
## Regeneration and integrity
11+
12+
[`generate-fixtures.py`](generate-fixtures.py) contains the complete document sources and prints each generated file's SHA-256 hash. Run it from the repository root:
13+
14+
```sh
15+
python3 src/integrations/misc/__tests__/fixtures/generate-fixtures.py
16+
```
17+
18+
Without an argument it writes beside the script, regardless of the current working directory. Supply an output directory as the optional argument to regenerate elsewhere for comparison. The generator uses only Python 3's standard library; it is not invoked by the test suite, so running tests does not require Python.
19+
20+
PDF object order and offsets, DOCX entry order, ZIP timestamps, Unix file permissions, and DEFLATE level 6 are fixed. The committed files have these sizes and SHA-256 hashes:
21+
22+
| Fixture | Bytes | SHA-256 |
23+
| ------------------------------ | ----: | ------------------------------------------------------------------ |
24+
| [approved.pdf](approved.pdf) | 643 | `11b19aed8bc2e40b9eda9d18b7bfe36e0bbc41c4383d7e30141927992790f411` |
25+
| [approved.docx](approved.docx) | 1002 | `732d827ad38ff51c24983c773bcb964ba53ca29bb055ea73aa9fbed820ae3a9b` |
26+
27+
The integration suite checks these hashes as well as the real parsers' extracted text. If a fixture is intentionally changed, update its generator source, the documented hash, and the corresponding test expectations together.

‎src/integrations/misc/__tests__/fixtures/generate-fixtures.py‎

Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
"""Regenerate the owned PDF/DOCX fixtures using only Python's standard library."""
2+
3+
import argparse
4+
import hashlib
5+
import io
6+
from pathlib import Path
7+
import zipfile
8+
9+
10+
def generate_pdf() -> bytes:
11+
stream = (
12+
b"BT\n/F1 12 Tf\n72 720 Td\n"
13+
b"(Approved PDF first line) Tj\n0 -18 Td\n"
14+
b"(Approved PDF second line) Tj\nET\n"
15+
)
16+
objects = [
17+
b"<< /Type /Catalog /Pages 2 0 R >>",
18+
b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>",
19+
b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] "
20+
b"/Resources << /Font << /F1 4 0 R >> >> /Contents 5 0 R >>",
21+
b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>",
22+
b"<< /Length %d >>\nstream\n" % len(stream) + stream + b"endstream",
23+
]
24+
document = bytearray(b"%PDF-1.4\n%\xe2\xe3\xcf\xd3\n")
25+
offsets = []
26+
for number, content in enumerate(objects, 1):
27+
offsets.append(len(document))
28+
document += b"%d 0 obj\n" % number + content + b"\nendobj\n"
29+
30+
xref_offset = len(document)
31+
document += b"xref\n0 6\n0000000000 65535 f \n"
32+
for offset in offsets:
33+
document += b"%010d 00000 n \n" % offset
34+
document += (
35+
b"trailer\n<< /Size 6 /Root 1 0 R >>\nstartxref\n%d\n%%%%EOF\n"
36+
% xref_offset
37+
)
38+
return bytes(document)
39+
40+
41+
def generate_docx() -> bytes:
42+
header = '<?xml version="1.0" encoding="UTF-8" standalone="yes"?>\n'
43+
parts = [
44+
(
45+
"[Content_Types].xml",
46+
header
47+
+ '<Types xmlns="http://schemas.openxmlformats.org/package/2006/content-types">\n'
48+
'<Default Extension="rels" ContentType="application/vnd.openxmlformats-package.relationships+xml"/>\n'
49+
'<Default Extension="xml" ContentType="application/xml"/>\n'
50+
'<Override PartName="/word/document.xml" '
51+
'ContentType="application/vnd.openxmlformats-officedocument.wordprocessingml.document.main+xml"/>\n'
52+
'</Types>',
53+
),
54+
(
55+
"_rels/.rels",
56+
header
57+
+ '<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">\n'
58+
'<Relationship Id="rId1" '
59+
'Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/officeDocument" '
60+
'Target="word/document.xml"/>\n'
61+
'</Relationships>',
62+
),
63+
(
64+
"word/document.xml",
65+
header
66+
+ '<w:document xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main">\n'
67+
'<w:body>\n'
68+
'<w:p><w:r><w:t>Approved DOCX first paragraph</w:t></w:r></w:p>\n'
69+
'<w:p><w:r><w:t xml:space="preserve">Approved DOCX second &amp; </w:t></w:r>'
70+
'<w:r><w:t>final paragraph</w:t></w:r></w:p>\n'
71+
'<w:sectPr><w:pgSz w:w="12240" w:h="15840"/></w:sectPr>\n'
72+
'</w:body>\n</w:document>',
73+
),
74+
]
75+
76+
output = io.BytesIO()
77+
with zipfile.ZipFile(output, "w") as archive:
78+
for name, content in parts:
79+
entry = zipfile.ZipInfo(name, (2020, 1, 1, 0, 0, 0))
80+
entry.compress_type = zipfile.ZIP_DEFLATED
81+
entry.create_system = 3
82+
entry.external_attr = 0o100644 << 16
83+
archive.writestr(entry, content.encode("utf-8"), compresslevel=6)
84+
return output.getvalue()
85+
86+
87+
def main() -> None:
88+
parser = argparse.ArgumentParser(description=__doc__)
89+
parser.add_argument(
90+
"output_directory",
91+
nargs="?",
92+
type=Path,
93+
default=Path(__file__).resolve().parent,
94+
help="destination directory (defaults to this script's directory)",
95+
)
96+
output_directory = parser.parse_args().output_directory
97+
output_directory.mkdir(parents=True, exist_ok=True)
98+
99+
for name, content in (
100+
("approved.pdf", generate_pdf()),
101+
("approved.docx", generate_docx()),
102+
):
103+
(output_directory / name).write_bytes(content)
104+
print(f"{hashlib.sha256(content).hexdigest()} {name}")
105+
106+
107+
if __name__ == "__main__":
108+
main()

0 commit comments

Comments
 (0)