Skip to content

Commit eaed70a

Browse files
roomoteroomote
committed
refactor(ci): isolate cached WASM test assets
1 parent 17aaaef commit eaed70a

9 files changed

Lines changed: 50 additions & 202 deletions

File tree

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ mock/
1717
# Builds
1818
bin/
1919
*.vsix
20+
/src/generated/
2021

2122
# Local prompts and rules
2223
/local-prompts

‎src/scripts/copy-tree-sitter-wasms.mjs‎

Lines changed: 6 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -5,72 +5,26 @@ import { fileURLToPath } from "node:url"
55

66
const srcDir = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..")
77
const wasmDir = path.join(srcDir, "node_modules", "tree-sitter-wasms", "out")
8-
const distDir = path.join(srcDir, "dist")
8+
const generatedDir = path.join(srcDir, "generated", "tree-sitter-wasms")
99
const wasmPattern = /^tree-sitter-.*\.wasm$/
1010

11-
export async function publishTreeSitterWasms(
12-
sourceDir,
13-
destinationDir,
14-
{ filesystem = fs.promises, onStep = async () => {}, signalState = { requested: undefined } } = {},
15-
) {
16-
const stagingDir = `${destinationDir}.tree-sitter-wasms-staging`
11+
export async function prepareTreeSitterWasms(sourceDir, destinationDir, { filesystem = fs.promises } = {}) {
1712
const sourceFiles = (await filesystem.readdir(sourceDir)).filter((filename) => wasmPattern.test(filename)).sort()
1813
if (sourceFiles.length === 0) throw new Error("WASM source set is empty")
19-
const checkpoint = async (name, filename) => {
20-
await onStep(name, filename)
21-
if (signalState.requested) throw new Error("WASM publication cancelled")
22-
}
2314

24-
await filesystem.rm(stagingDir, { recursive: true, force: true })
25-
await filesystem.mkdir(stagingDir, { recursive: true })
15+
await filesystem.rm(destinationDir, { recursive: true, force: true })
2616
await filesystem.mkdir(destinationDir, { recursive: true })
2717
try {
2818
for (const filename of sourceFiles) {
29-
await filesystem.copyFile(path.join(sourceDir, filename), path.join(stagingDir, filename))
30-
await checkpoint("staged", filename)
31-
}
32-
for (const filename of await filesystem.readdir(stagingDir)) {
33-
if (!wasmPattern.test(filename)) throw new Error(`Unexpected staged WASM: ${filename}`)
34-
}
35-
await checkpoint("validated", stagingDir)
36-
37-
for (const filename of await filesystem.readdir(destinationDir)) {
38-
if (wasmPattern.test(filename) || (filename.includes("tree-sitter-") && filename.endsWith(".tmp"))) {
39-
await filesystem.rm(path.join(destinationDir, filename), { force: true })
40-
}
41-
}
42-
for (const filename of sourceFiles) {
43-
await filesystem.rename(path.join(stagingDir, filename), path.join(destinationDir, filename))
44-
await checkpoint("published", filename)
19+
await filesystem.copyFile(path.join(sourceDir, filename), path.join(destinationDir, filename))
4520
}
46-
await checkpoint("completed", destinationDir)
4721
} catch (error) {
48-
try {
49-
for (const filename of await filesystem.readdir(destinationDir)) {
50-
if (wasmPattern.test(filename))
51-
await filesystem.rm(path.join(destinationDir, filename), { force: true })
52-
}
53-
} catch (cleanupError) {
54-
throw new AggregateError(
55-
[error, cleanupError],
56-
`WASM publication and cleanup failed; remove ${stagingDir} and rerun to rebuild`,
57-
)
58-
}
22+
await filesystem.rm(destinationDir, { recursive: true, force: true })
5923
throw error
60-
} finally {
61-
await filesystem.rm(stagingDir, { recursive: true, force: true })
6224
}
6325
return { sourceFiles }
6426
}
6527

6628
if (process.argv[1] === fileURLToPath(import.meta.url)) {
67-
const signalState = { requested: undefined }
68-
for (const signal of ["SIGINT", "SIGTERM"]) process.on(signal, () => (signalState.requested ??= signal))
69-
try {
70-
await publishTreeSitterWasms(wasmDir, distDir, { signalState })
71-
if (signalState.requested) process.exitCode = signalState.requested === "SIGINT" ? 130 : 143
72-
} catch (error) {
73-
if (!signalState.requested) throw error
74-
process.exitCode = signalState.requested === "SIGINT" ? 130 : 143
75-
}
29+
await prepareTreeSitterWasms(wasmDir, generatedDir)
7630
}

‎src/scripts/copy-tree-sitter-wasms.spec.mjs‎

Lines changed: 22 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -3,85 +3,55 @@ import os from "node:os"
33
import path from "node:path"
44
import { afterEach, beforeEach, describe, expect, it } from "vitest"
55

6-
import { publishTreeSitterWasms } from "./copy-tree-sitter-wasms.mjs"
6+
import { prepareTreeSitterWasms } from "./copy-tree-sitter-wasms.mjs"
77

8-
describe("publishTreeSitterWasms", () => {
8+
describe("prepareTreeSitterWasms", () => {
99
let root
1010
let source
1111
let destination
1212

1313
beforeEach(() => {
1414
root = fs.mkdtempSync(path.join(os.tmpdir(), "tree-sitter-wasms-"))
1515
source = path.join(root, "source")
16-
destination = path.join(root, "dist")
16+
destination = path.join(root, "generated", "tree-sitter-wasms")
1717
fs.mkdirSync(source)
18-
fs.mkdirSync(destination)
1918
fs.writeFileSync(path.join(source, "tree-sitter-a.wasm"), "a")
2019
})
2120

2221
afterEach(() => fs.rmSync(root, { recursive: true, force: true }))
2322

24-
it("publishes the exact staged set and removes stale state", async () => {
23+
it("replaces pre-existing generated output without filesystem rename", async () => {
2524
fs.writeFileSync(path.join(source, "ignored.txt"), "ignored")
25+
fs.mkdirSync(destination, { recursive: true })
2626
fs.writeFileSync(path.join(destination, "tree-sitter-stale.wasm"), "stale")
27-
fs.mkdirSync(`${destination}.tree-sitter-wasms-staging`)
28-
fs.writeFileSync(path.join(`${destination}.tree-sitter-wasms-staging`, "stale.tmp"), "stale")
29-
30-
await publishTreeSitterWasms(source, destination)
31-
32-
expect(fs.readdirSync(destination)).toEqual(["tree-sitter-a.wasm"])
33-
expect(fs.readFileSync(path.join(destination, "tree-sitter-a.wasm"), "utf8")).toBe("a")
34-
expect(fs.existsSync(`${destination}.tree-sitter-wasms-staging`)).toBe(false)
35-
})
36-
37-
it("fails without publishing when staging fails", async () => {
3827
const filesystem = {
3928
...fs.promises,
40-
copyFile: async () => {
41-
throw new Error("copy failed")
29+
rename: async () => {
30+
throw new Error("rename must not be used")
4231
},
4332
}
44-
await expect(publishTreeSitterWasms(source, destination, { filesystem })).rejects.toThrow("copy failed")
45-
expect(fs.readdirSync(destination)).toEqual([])
46-
})
4733

48-
it("removes partial outputs and fails when publication is interrupted", async () => {
49-
fs.writeFileSync(path.join(source, "tree-sitter-b.wasm"), "b")
50-
const signalState = { requested: undefined }
51-
await expect(
52-
publishTreeSitterWasms(source, destination, {
53-
signalState,
54-
onStep(name) {
55-
if (name === "published") signalState.requested = "SIGTERM"
56-
},
57-
}),
58-
).rejects.toThrow("WASM publication cancelled")
59-
expect(fs.readdirSync(destination)).toEqual([])
34+
await prepareTreeSitterWasms(source, destination, { filesystem })
35+
36+
expect(fs.readdirSync(destination)).toEqual(["tree-sitter-a.wasm"])
37+
expect(fs.readFileSync(path.join(destination, "tree-sitter-a.wasm"), "utf8")).toBe("a")
6038
})
6139

62-
it("reports publication and cleanup errors together", async () => {
63-
let destinationReads = 0
40+
it("removes partial task output after a copy failure and rebuilds cleanly", async () => {
41+
fs.writeFileSync(path.join(source, "tree-sitter-b.wasm"), "b")
42+
let copies = 0
6443
const filesystem = {
6544
...fs.promises,
66-
readdir(directory) {
67-
if (directory === destination && ++destinationReads === 2) throw new Error("cleanup enumeration failed")
68-
return fs.promises.readdir(directory)
45+
copyFile: async (...args) => {
46+
if (++copies === 2) throw new Error("copy failed")
47+
return fs.promises.copyFile(...args)
6948
},
7049
}
7150

72-
await expect(
73-
publishTreeSitterWasms(source, destination, {
74-
filesystem,
75-
onStep(name) {
76-
if (name === "published") throw new Error("publication failed")
77-
},
78-
}),
79-
).rejects.toMatchObject({
80-
message: expect.stringContaining("WASM publication and cleanup failed"),
81-
errors: [
82-
expect.objectContaining({ message: "publication failed" }),
83-
expect.objectContaining({ message: "cleanup enumeration failed" }),
84-
],
85-
})
51+
await expect(prepareTreeSitterWasms(source, destination, { filesystem })).rejects.toThrow("copy failed")
52+
expect(fs.existsSync(destination)).toBe(false)
53+
54+
await prepareTreeSitterWasms(source, destination)
55+
expect(fs.readdirSync(destination)).toEqual(["tree-sitter-a.wasm", "tree-sitter-b.wasm"])
8656
})
8757
})

‎src/scripts/dist-sandbox.mjs‎

Lines changed: 0 additions & 18 deletions
This file was deleted.

‎src/scripts/dist-sandbox.spec.mjs‎

Lines changed: 0 additions & 40 deletions
This file was deleted.

‎src/scripts/verify-coverage-contract.mjs‎

Lines changed: 12 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@ import process from "node:process"
55
import { fileURLToPath } from "node:url"
66

77
import { assertMatchingFiles } from "./verify-wasm-files.mjs"
8-
import { createDistSandbox } from "./dist-sandbox.mjs"
98

109
const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..")
1110
const pnpm = process.platform === "win32" ? process.env.npm_execpath : "pnpm"
@@ -37,24 +36,12 @@ const preparationTask = graph.tasks.find(({ taskId }) => taskId === "zoo-code#pr
3736
if (!coverageTask?.dependencies.includes("zoo-code#prepare:tree-sitter-wasms"))
3837
throw new Error("WASM prerequisite missing")
3938
if (coverageTask.dependencies.includes("zoo-code#bundle")) throw new Error("Unit coverage must not depend on bundle")
40-
if (JSON.stringify(preparationTask?.outputs) !== JSON.stringify(["dist/tree-sitter-*.wasm"]))
39+
if (JSON.stringify(preparationTask?.outputs) !== JSON.stringify(["generated/tree-sitter-wasms/**"]))
4140
throw new Error("WASM prerequisite outputs changed")
4241

43-
const dist = path.join(root, "src", "dist")
42+
const generated = path.join(root, "src", "generated", "tree-sitter-wasms")
4443
const cacheDir = path.join(root, ".turbo", "coverage-contract")
4544
fs.rmSync(cacheDir, { recursive: true, force: true })
46-
const state = { sandbox: undefined }
47-
for (const signal of ["SIGINT", "SIGTERM"]) {
48-
process.once(signal, () => {
49-
try {
50-
state.sandbox?.restore()
51-
} finally {
52-
fs.rmSync(cacheDir, { recursive: true, force: true })
53-
}
54-
process.exit(1)
55-
})
56-
}
57-
state.sandbox = createDistSandbox(dist)
5845

5946
try {
6047
run(["turbo", "run", "prepare:tree-sitter-wasms", "--filter=zoo-code", "--cache-dir=.turbo/coverage-contract"])
@@ -64,23 +51,21 @@ try {
6451
.readdirSync(path.join(root, "src", "node_modules", "tree-sitter-wasms", "out"))
6552
.filter((filename) => /^tree-sitter-.*\.wasm$/.test(filename))
6653
.sort()
67-
const published = fs
68-
.readdirSync(dist)
54+
const prepared = fs
55+
.readdirSync(generated)
6956
.filter((filename) => /^tree-sitter-.*\.wasm$/.test(filename))
7057
.sort()
7158
if (source.length === 0) throw new Error("Dependency contains no tree-sitter WASMs")
72-
if (JSON.stringify(source) !== JSON.stringify(published))
73-
throw new Error("Published WASM set does not match dependency")
59+
if (JSON.stringify(source) !== JSON.stringify(prepared))
60+
throw new Error("Prepared WASM set does not match dependency")
7461
assertMatchingFiles(
7562
path.join(root, "src", "node_modules", "tree-sitter-wasms", "out"),
76-
dist,
63+
generated,
7764
source,
78-
"Published WASM content does not match dependency",
65+
"Prepared WASM content does not match dependency",
7966
)
80-
if (fs.readdirSync(dist).some((filename) => filename.endsWith(".tmp")))
81-
throw new Error("Temporary WASM files remain")
8267

83-
for (const filename of published) fs.rmSync(path.join(dist, filename), { force: true })
68+
fs.rmSync(generated, { recursive: true, force: true })
8469
const warmGraph = JSON.parse(
8570
run(
8671
[
@@ -98,20 +83,16 @@ try {
9883
if (warmTask?.cache.status !== "HIT") throw new Error("WASM prerequisite is not available in the isolated cache")
9984
run(["turbo", "run", "prepare:tree-sitter-wasms", "--filter=zoo-code", "--cache-dir=.turbo/coverage-contract"])
10085
const restored = fs
101-
.readdirSync(dist)
86+
.readdirSync(generated)
10287
.filter((filename) => /^tree-sitter-.*\.wasm$/.test(filename))
10388
.sort()
10489
if (JSON.stringify(source) !== JSON.stringify(restored)) throw new Error("WASM cache did not restore exact outputs")
10590
assertMatchingFiles(
10691
path.join(root, "src", "node_modules", "tree-sitter-wasms", "out"),
107-
dist,
92+
generated,
10893
source,
10994
"WASM cache restored corrupted output",
11095
)
11196
} finally {
112-
try {
113-
state.sandbox.restore()
114-
} finally {
115-
fs.rmSync(cacheDir, { recursive: true, force: true })
116-
}
97+
fs.rmSync(cacheDir, { recursive: true, force: true })
11798
}

‎src/services/tree-sitter/__tests__/helpers.ts‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -34,12 +34,12 @@ export async function initializeTreeSitter() {
3434
// Initialize directly using the default export or the module itself
3535
await Parser.init()
3636

37-
// Override the Parser.Language.load to use dist directory
37+
// Use the cacheable test prerequisite rather than the bundled extension output.
3838
const originalLoad = Language.load
3939

4040
Language.load = async (wasmPath: string) => {
4141
const filename = path.basename(wasmPath)
42-
const correctPath = path.join(process.cwd(), "dist", filename)
42+
const correctPath = path.join(process.cwd(), "generated", "tree-sitter-wasms", filename)
4343
// console.log(`Redirecting WASM load from ${wasmPath} to ${correctPath}`)
4444
return originalLoad(correctPath)
4545
}
@@ -84,7 +84,7 @@ export async function testParseSourceCodeDefinitions(
8484
const parser = new Parser()
8585

8686
// Load language and configure parser
87-
const wasmPath = path.join(process.cwd(), `dist/${wasmFile}`)
87+
const wasmPath = path.join(process.cwd(), "generated", "tree-sitter-wasms", wasmFile)
8888
const lang = await Language.load(wasmPath)
8989
parser.setLanguage(lang)
9090

@@ -113,7 +113,7 @@ export async function testParseSourceCodeDefinitions(
113113
export async function inspectTreeStructure(content: string, language: string = "typescript"): Promise<string> {
114114
const { Parser, Language } = await initializeTreeSitter()
115115
const parser = new Parser()
116-
const wasmPath = path.join(process.cwd(), `dist/tree-sitter-${language}.wasm`)
116+
const wasmPath = path.join(process.cwd(), "generated", "tree-sitter-wasms", `tree-sitter-${language}.wasm`)
117117
const lang = await Language.load(wasmPath)
118118
parser.setLanguage(lang)
119119

‎src/services/tree-sitter/__tests__/languageParser.spec.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ import * as path from "path"
44
import { loadRequiredLanguageParsers } from "../languageParser"
55

66
// Path to the directory containing the WASM files.
7-
const WASM_DIR = path.join(__dirname, "../../../node_modules/tree-sitter-wasms/out")
7+
const WASM_DIR = path.join(__dirname, "../../../generated/tree-sitter-wasms")
88

99
describe("loadRequiredLanguageParsers", () => {
1010
it("should load Python parser for .py files", async () => {

0 commit comments

Comments
 (0)