Skip to content

Commit a58036a

Browse files
committed
refactor(ci): load test WASMs from dependency
1 parent eaed70a commit a58036a

12 files changed

Lines changed: 111 additions & 243 deletions

‎.gitignore‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@ mock/
1717
# Builds
1818
bin/
1919
*.vsix
20-
/src/generated/
2120

2221
# Local prompts and rules
2322
/local-prompts

‎src/package.json‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -442,7 +442,6 @@
442442
"lint": "eslint . --ext=ts --max-warnings=0",
443443
"check-types": "tsc --noEmit",
444444
"test": "vitest run",
445-
"prepare:tree-sitter-wasms": "node scripts/copy-tree-sitter-wasms.mjs",
446445
"verify:coverage-contract": "node scripts/verify-coverage-contract.mjs",
447446
"test:unit": "vitest run --config vitest.unit.config.ts",
448447
"test:dist": "vitest run --config vitest.dist.config.ts",

‎src/scripts/copy-tree-sitter-wasms.mjs‎

Lines changed: 0 additions & 30 deletions
This file was deleted.

‎src/scripts/copy-tree-sitter-wasms.spec.mjs‎

Lines changed: 0 additions & 57 deletions
This file was deleted.
Lines changed: 18 additions & 89 deletions
Original file line numberDiff line numberDiff line change
@@ -1,98 +1,27 @@
11
import { spawnSync } from "node:child_process"
2-
import fs from "node:fs"
3-
import path from "node:path"
42
import process from "node:process"
5-
import { fileURLToPath } from "node:url"
63

7-
import { assertMatchingFiles } from "./verify-wasm-files.mjs"
8-
9-
const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..")
104
const pnpm = process.platform === "win32" ? process.env.npm_execpath : "pnpm"
115
if (!pnpm) throw new Error("pnpm executable path is unavailable")
12-
const pnpmPrefix = process.platform === "win32" ? [pnpm] : []
13-
const run = (args, options = {}) => {
14-
const { includeStderr = true, ...spawnOptions } = options
15-
const command = process.platform === "win32" ? process.execPath : pnpm
16-
const result = spawnSync(command, [...pnpmPrefix, ...args], { cwd: root, encoding: "utf8", ...spawnOptions })
17-
if (result.status !== 0) {
18-
const details = [
19-
result.error?.message,
20-
result.signal ? `terminated by ${result.signal}` : undefined,
21-
result.stderr,
22-
result.stdout,
23-
]
24-
.filter(Boolean)
25-
.join("\n")
26-
throw new Error(details || `pnpm exited with status ${result.status ?? "unknown"}`)
27-
}
28-
return `${result.stdout || ""}${includeStderr ? result.stderr || "" : ""}`
6+
const command = process.platform === "win32" ? process.execPath : pnpm
7+
const args = process.platform === "win32" ? [pnpm] : []
8+
const result = spawnSync(
9+
command,
10+
[...args, "turbo", "run", "test:coverage:unit", "test:dist", "--filter=zoo-code", "--dry=json"],
11+
{ encoding: "utf8" },
12+
)
13+
if (result.status !== 0) {
14+
const details = [result.error?.message, result.signal, result.stderr, result.stdout].filter(Boolean).join("\n")
15+
throw new Error(details || `pnpm exited with status ${result.status ?? "unknown"}`)
2916
}
3017

31-
const graph = JSON.parse(
32-
run(["turbo", "run", "test:coverage:unit", "--filter=zoo-code", "--dry=json"], { includeStderr: false }),
33-
)
18+
const graph = JSON.parse(result.stdout)
3419
const coverageTask = graph.tasks.find(({ taskId }) => taskId === "zoo-code#test:coverage:unit")
35-
const preparationTask = graph.tasks.find(({ taskId }) => taskId === "zoo-code#prepare:tree-sitter-wasms")
36-
if (!coverageTask?.dependencies.includes("zoo-code#prepare:tree-sitter-wasms"))
37-
throw new Error("WASM prerequisite missing")
20+
const distTask = graph.tasks.find(({ taskId }) => taskId === "zoo-code#test:dist")
21+
if (!coverageTask) throw new Error("Unit coverage task missing")
22+
if (graph.tasks.some(({ taskId }) => taskId === "zoo-code#prepare:tree-sitter-wasms"))
23+
throw new Error("Removed WASM preparation task remains in the graph")
3824
if (coverageTask.dependencies.includes("zoo-code#bundle")) throw new Error("Unit coverage must not depend on bundle")
39-
if (JSON.stringify(preparationTask?.outputs) !== JSON.stringify(["generated/tree-sitter-wasms/**"]))
40-
throw new Error("WASM prerequisite outputs changed")
41-
42-
const generated = path.join(root, "src", "generated", "tree-sitter-wasms")
43-
const cacheDir = path.join(root, ".turbo", "coverage-contract")
44-
fs.rmSync(cacheDir, { recursive: true, force: true })
45-
46-
try {
47-
run(["turbo", "run", "prepare:tree-sitter-wasms", "--filter=zoo-code", "--cache-dir=.turbo/coverage-contract"])
48-
run(["--dir", "src", "exec", "vitest", "run", "services/tree-sitter/__tests__"], { stdio: "inherit" })
49-
50-
const source = fs
51-
.readdirSync(path.join(root, "src", "node_modules", "tree-sitter-wasms", "out"))
52-
.filter((filename) => /^tree-sitter-.*\.wasm$/.test(filename))
53-
.sort()
54-
const prepared = fs
55-
.readdirSync(generated)
56-
.filter((filename) => /^tree-sitter-.*\.wasm$/.test(filename))
57-
.sort()
58-
if (source.length === 0) throw new Error("Dependency contains no tree-sitter WASMs")
59-
if (JSON.stringify(source) !== JSON.stringify(prepared))
60-
throw new Error("Prepared WASM set does not match dependency")
61-
assertMatchingFiles(
62-
path.join(root, "src", "node_modules", "tree-sitter-wasms", "out"),
63-
generated,
64-
source,
65-
"Prepared WASM content does not match dependency",
66-
)
67-
68-
fs.rmSync(generated, { recursive: true, force: true })
69-
const warmGraph = JSON.parse(
70-
run(
71-
[
72-
"turbo",
73-
"run",
74-
"prepare:tree-sitter-wasms",
75-
"--filter=zoo-code",
76-
"--cache-dir=.turbo/coverage-contract",
77-
"--dry=json",
78-
],
79-
{ includeStderr: false },
80-
),
81-
)
82-
const warmTask = warmGraph.tasks.find(({ taskId }) => taskId === "zoo-code#prepare:tree-sitter-wasms")
83-
if (warmTask?.cache.status !== "HIT") throw new Error("WASM prerequisite is not available in the isolated cache")
84-
run(["turbo", "run", "prepare:tree-sitter-wasms", "--filter=zoo-code", "--cache-dir=.turbo/coverage-contract"])
85-
const restored = fs
86-
.readdirSync(generated)
87-
.filter((filename) => /^tree-sitter-.*\.wasm$/.test(filename))
88-
.sort()
89-
if (JSON.stringify(source) !== JSON.stringify(restored)) throw new Error("WASM cache did not restore exact outputs")
90-
assertMatchingFiles(
91-
path.join(root, "src", "node_modules", "tree-sitter-wasms", "out"),
92-
generated,
93-
source,
94-
"WASM cache restored corrupted output",
95-
)
96-
} finally {
97-
fs.rmSync(cacheDir, { recursive: true, force: true })
98-
}
25+
if (!Object.hasOwn(coverageTask.inputs, "package.json")) throw new Error("Unit coverage must hash package.json")
26+
if (!coverageTask.hashOfExternalDependencies) throw new Error("Unit coverage must hash external dependencies")
27+
if (!distTask?.dependencies.includes("zoo-code#bundle")) throw new Error("Dist smoke test must depend on bundle")

‎src/scripts/verify-wasm-files.mjs‎

Lines changed: 0 additions & 14 deletions
This file was deleted.

‎src/scripts/verify-wasm-files.spec.mjs‎

Lines changed: 0 additions & 29 deletions
This file was deleted.

‎src/services/tree-sitter/__tests__/helpers.ts‎

Lines changed: 4 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@ import * as path from "path"
44
import tsxQuery from "../queries/tsx"
55
import { Parser, Language } from "web-tree-sitter"
66

7+
import { loadTestGrammar } from "./wasm"
8+
79
vi.mock("fs/promises")
810
export const mockedFs = vi.mocked(fs)
911

@@ -34,16 +36,6 @@ export async function initializeTreeSitter() {
3436
// Initialize directly using the default export or the module itself
3537
await Parser.init()
3638

37-
// Use the cacheable test prerequisite rather than the bundled extension output.
38-
const originalLoad = Language.load
39-
40-
Language.load = async (wasmPath: string) => {
41-
const filename = path.basename(wasmPath)
42-
const correctPath = path.join(process.cwd(), "generated", "tree-sitter-wasms", filename)
43-
// console.log(`Redirecting WASM load from ${wasmPath} to ${correctPath}`)
44-
return originalLoad(correctPath)
45-
}
46-
4739
initializedTreeSitter = { Parser, Language }
4840
}
4941

@@ -84,8 +76,7 @@ export async function testParseSourceCodeDefinitions(
8476
const parser = new Parser()
8577

8678
// Load language and configure parser
87-
const wasmPath = path.join(process.cwd(), "generated", "tree-sitter-wasms", wasmFile)
88-
const lang = await Language.load(wasmPath)
79+
const lang = await loadTestGrammar(path.basename(wasmFile))
8980
parser.setLanguage(lang)
9081

9182
// Create a real query
@@ -113,8 +104,7 @@ export async function testParseSourceCodeDefinitions(
113104
export async function inspectTreeStructure(content: string, language: string = "typescript"): Promise<string> {
114105
const { Parser, Language } = await initializeTreeSitter()
115106
const parser = new Parser()
116-
const wasmPath = path.join(process.cwd(), "generated", "tree-sitter-wasms", `tree-sitter-${language}.wasm`)
117-
const lang = await Language.load(wasmPath)
107+
const lang = await loadTestGrammar(`tree-sitter-${language}.wasm`)
118108
parser.setLanguage(lang)
119109

120110
// Parse the content

‎src/services/tree-sitter/__tests__/languageParser.spec.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ import * as path from "path"
44
import { loadRequiredLanguageParsers } from "../languageParser"
55

66
// Path to the directory containing the WASM files.
7-
const WASM_DIR = path.join(__dirname, "../../../generated/tree-sitter-wasms")
7+
const WASM_DIR = path.join(__dirname, "../../../node_modules/tree-sitter-wasms/out")
88

99
describe("loadRequiredLanguageParsers", () => {
1010
it("should load Python parser for .py files", async () => {
Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
import fs from "fs"
2+
import os from "os"
3+
import path from "path"
4+
import { afterEach, beforeAll, describe, expect, it } from "vitest"
5+
import { Parser } from "web-tree-sitter"
6+
7+
import { loadTestGrammar } from "./wasm"
8+
9+
const requiredGrammars = [
10+
"c",
11+
"cpp",
12+
"c_sharp",
13+
"css",
14+
"dart",
15+
"elisp",
16+
"elixir",
17+
"embedded_template",
18+
"go",
19+
"html",
20+
"java",
21+
"javascript",
22+
"json",
23+
"kotlin",
24+
"lua",
25+
"ocaml",
26+
"php",
27+
"python",
28+
"ruby",
29+
"rust",
30+
"scala",
31+
"solidity",
32+
"swift",
33+
"systemrdl",
34+
"tlaplus",
35+
"toml",
36+
"tsx",
37+
"typescript",
38+
"vue",
39+
"zig",
40+
]
41+
42+
describe("dependency-owned Tree-sitter grammars", () => {
43+
const temporaryDirectories: string[] = []
44+
45+
beforeAll(() => Parser.init())
46+
afterEach(() => temporaryDirectories.splice(0).forEach((directory) => fs.rmSync(directory, { recursive: true })))
47+
48+
it.each(requiredGrammars)("loads tree-sitter-%s.wasm", async (grammar) => {
49+
await expect(loadTestGrammar(`tree-sitter-${grammar}.wasm`)).resolves.toBeDefined()
50+
})
51+
52+
it("reports a missing dependency artifact with its filename and resolved path", async () => {
53+
const directory = fs.mkdtempSync(path.join(os.tmpdir(), "tree-sitter-wasm-missing-"))
54+
temporaryDirectories.push(directory)
55+
56+
await expect(loadTestGrammar("tree-sitter-missing.wasm", directory)).rejects.toThrow(
57+
`Failed to load Tree-sitter grammar tree-sitter-missing.wasm from ${path.join(directory, "tree-sitter-missing.wasm")}`,
58+
)
59+
})
60+
61+
it("reports a malformed dependency artifact with its filename and resolved path", async () => {
62+
const directory = fs.mkdtempSync(path.join(os.tmpdir(), "tree-sitter-wasm-malformed-"))
63+
temporaryDirectories.push(directory)
64+
fs.writeFileSync(path.join(directory, "tree-sitter-malformed.wasm"), "not wasm")
65+
66+
await expect(loadTestGrammar("tree-sitter-malformed.wasm", directory)).rejects.toThrow(
67+
`Failed to load Tree-sitter grammar tree-sitter-malformed.wasm from ${path.join(directory, "tree-sitter-malformed.wasm")}`,
68+
)
69+
})
70+
})

0 commit comments

Comments
 (0)