Skip to content

Commit 43159fe

Browse files
daewoongohdaewoongoh
committed
fix(scripts): validate --build-mode and add tests for the gray-screen tools
Only build into the fixed production/development temp directories (Vite empties the output directory), never into a symlink. Add subprocess and HTTP tests for generate-large-task, analyze-session and mock-openai-server, and make analyze-session skip tasks whose ui_messages.json cannot be parsed.
1 parent ae5662a commit 43159fe

5 files changed

Lines changed: 246 additions & 4 deletions

File tree

‎scripts/gray-screen/__tests__/lib.test.mjs‎

Lines changed: 28 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,14 @@ import os from "node:os"
44
import path from "node:path"
55
import { afterEach, beforeEach, describe, it } from "node:test"
66

7-
import { integerFlag, parseFlagArgs, resolveServedFile, validateRelativeDir, writeTaskAtomically } from "../lib.mjs"
7+
import {
8+
integerFlag,
9+
parseFlagArgs,
10+
resolveBuildDir,
11+
resolveServedFile,
12+
validateRelativeDir,
13+
writeTaskAtomically,
14+
} from "../lib.mjs"
815

916
let tmp
1017

@@ -46,6 +53,26 @@ describe("validateRelativeDir", () => {
4653
})
4754
})
4855

56+
describe("resolveBuildDir", () => {
57+
it("maps the documented modes to fixed directories under the temp root", () => {
58+
assert.equal(resolveBuildDir("production", tmp), path.join(tmp, "zoo-webview-stress-build"))
59+
assert.equal(resolveBuildDir("development", tmp), path.join(tmp, "zoo-webview-stress-build-development"))
60+
})
61+
62+
it("rejects any other mode, including path traversal", () => {
63+
for (const mode of ["../../../tmp/victim", "staging", "", "production/../x"]) {
64+
assert.throws(() => resolveBuildDir(mode, tmp), /Invalid --build-mode/, mode)
65+
}
66+
})
67+
68+
it("refuses an output path that is a symlink", () => {
69+
fs.mkdirSync(path.join(tmp, "elsewhere"))
70+
fs.symlinkSync(path.join(tmp, "elsewhere"), path.join(tmp, "zoo-webview-stress-build"))
71+
72+
assert.throws(() => resolveBuildDir("production", tmp), /symlink/)
73+
})
74+
})
75+
4976
describe("resolveServedFile", () => {
5077
beforeEach(() => {
5178
fs.mkdirSync(path.join(tmp, "build", "assets"), { recursive: true })

‎scripts/gray-screen/__tests__/tools.test.mjs‎

Lines changed: 183 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,183 @@
1+
// Subprocess tests for the executable gray-screen tools. Run: node --test scripts/gray-screen/__tests__/tools.test.mjs
2+
// The browser harnesses (webview-heap-matrix, webview-render-stress) need a built webview and Chromium, so only
3+
// their pure parts (static serving, build directory) are covered, in lib.test.mjs.
4+
import assert from "node:assert/strict"
5+
import { spawn, spawnSync } from "node:child_process"
6+
import fs from "node:fs"
7+
import net from "node:net"
8+
import os from "node:os"
9+
import path from "node:path"
10+
import { afterEach, beforeEach, describe, it } from "node:test"
11+
import { fileURLToPath } from "node:url"
12+
13+
const dir = path.dirname(fileURLToPath(import.meta.url))
14+
const script = (name) => path.join(dir, "..", name)
15+
const node = (name, args) => spawnSync(process.execPath, [script(name), ...args], { encoding: "utf8" })
16+
17+
let tmp
18+
19+
beforeEach(() => {
20+
tmp = fs.mkdtempSync(path.join(os.tmpdir(), "gray-screen-tools-"))
21+
})
22+
23+
afterEach(() => {
24+
fs.rmSync(tmp, { recursive: true, force: true })
25+
})
26+
27+
const generate = (...args) => node("generate-large-task.mjs", ["--storage", tmp, ...args])
28+
29+
function readTask() {
30+
const [id] = fs.readdirSync(path.join(tmp, "tasks"))
31+
const read = (name) => JSON.parse(fs.readFileSync(path.join(tmp, "tasks", id, name), "utf8"))
32+
return { id, messages: read("ui_messages.json"), api: read("api_conversation_history.json"), item: read("history_item.json") }
33+
}
34+
35+
describe("generate-large-task", () => {
36+
it("writes a complete task with the requested number of messages", () => {
37+
const result = generate("--messages", "40", "--text-bytes", "100")
38+
39+
assert.equal(result.status, 0, result.stderr)
40+
const { id, messages, item } = readTask()
41+
assert.ok(messages.length >= 40)
42+
assert.equal(item.id, id)
43+
assert.match(item.task, /\[LOAD TEST\]/)
44+
assert.deepEqual(fs.readdirSync(tmp).sort(), ["tasks"])
45+
})
46+
47+
it("adds images when asked to", () => {
48+
assert.equal(generate("--messages", "30", "--text-bytes", "100", "--image-every", "5", "--image-kb", "1").status, 0)
49+
50+
assert.ok(readTask().messages.some((m) => m.images?.length))
51+
})
52+
53+
it("produces non-Latin-1 text with --two-byte true only", () => {
54+
assert.equal(generate("--messages", "30", "--text-bytes", "100", "--two-byte", "true").status, 0)
55+
assert.match(JSON.stringify(readTask().messages), /한글/)
56+
57+
fs.rmSync(path.join(tmp, "tasks"), { recursive: true })
58+
assert.equal(generate("--messages", "30", "--text-bytes", "100").status, 0)
59+
assert.doesNotMatch(JSON.stringify(readTask().messages), /한글/)
60+
})
61+
62+
it("creates nothing when a flag has no value or is not a positive integer", () => {
63+
for (const args of [["--messages", "--text-bytes", "5"], ["--messages", "abc"], ["--messages", "0"], ["--image-kb", "-1"]]) {
64+
const result = generate(...args)
65+
66+
assert.notEqual(result.status, 0, args.join(" "))
67+
assert.deepEqual(fs.readdirSync(tmp), [], args.join(" "))
68+
}
69+
})
70+
})
71+
72+
describe("analyze-session", () => {
73+
const analyze = (...args) => node("analyze-session.mjs", ["--storage", tmp, ...args])
74+
75+
it("excludes generated [LOAD TEST] tasks unless asked, and reports sizes for the rest", () => {
76+
assert.equal(generate("--messages", "40", "--text-bytes", "100").status, 0)
77+
78+
const excluded = analyze()
79+
assert.equal(excluded.status, 0, excluded.stderr)
80+
assert.match(excluded.stdout, /excluded 1 generated/)
81+
assert.match(excluded.stdout, /analyzed set: 0/)
82+
83+
const included = analyze("--include-generated", "true")
84+
assert.equal(included.status, 0, included.stderr)
85+
assert.match(included.stdout, /analyzed set: 1/)
86+
assert.match(included.stdout, /size\(MB\)\s+msgs\s+toolTxt/)
87+
})
88+
89+
it("does not print message content", () => {
90+
const taskDir = path.join(tmp, "tasks", "real-task")
91+
fs.mkdirSync(taskDir, { recursive: true })
92+
fs.writeFileSync(
93+
path.join(taskDir, "ui_messages.json"),
94+
JSON.stringify([{ ts: 1, type: "say", say: "text", text: "TOP-SECRET-CONTENT" }]),
95+
)
96+
97+
const result = analyze()
98+
99+
assert.equal(result.status, 0, result.stderr)
100+
assert.doesNotMatch(result.stdout, /TOP-SECRET-CONTENT/)
101+
})
102+
103+
it("survives a malformed ui_messages.json", () => {
104+
const taskDir = path.join(tmp, "tasks", "broken")
105+
fs.mkdirSync(taskDir, { recursive: true })
106+
fs.writeFileSync(path.join(taskDir, "ui_messages.json"), "{not json")
107+
108+
assert.equal(analyze().status, 0)
109+
})
110+
})
111+
112+
describe("mock-openai-server", () => {
113+
it("rejects an unsafe --dir before listening", () => {
114+
for (const bad of ['x"; touch /tmp/pwned; #', ".", "../x", "-rf"]) {
115+
const result = node("mock-openai-server.mjs", ["--dir", bad, "--port", "0"])
116+
117+
assert.equal(result.status, 1, bad)
118+
assert.match(result.stderr, /Invalid --dir/)
119+
}
120+
})
121+
122+
describe("HTTP", () => {
123+
let child
124+
let base
125+
126+
beforeEach(async () => {
127+
const port = await new Promise((resolve) => {
128+
const probe = net.createServer().listen(0, "127.0.0.1", () => {
129+
const { port } = probe.address()
130+
probe.close(() => resolve(port))
131+
})
132+
})
133+
base = `http://127.0.0.1:${port}`
134+
child = spawn(
135+
process.execPath,
136+
[script("mock-openai-server.mjs"), "--scenario", "rapid", "--port", String(port), "--max-requests", "1"],
137+
{ stdio: ["ignore", "pipe", "inherit"] },
138+
)
139+
await new Promise((resolve, reject) => {
140+
child.once("error", reject)
141+
child.stdout.on("data", (chunk) => String(chunk).includes("listening") && resolve())
142+
})
143+
})
144+
145+
afterEach(() => child.kill())
146+
147+
const complete = (body) =>
148+
fetch(`${base}/v1/chat/completions`, {
149+
method: "POST",
150+
headers: { "content-type": "application/json" },
151+
body: JSON.stringify(body),
152+
})
153+
154+
it("lists the mock model and 404s elsewhere", async () => {
155+
const models = await (await fetch(`${base}/v1/models`)).json()
156+
assert.deepEqual(
157+
models.data.map((m) => m.id),
158+
["mock"],
159+
)
160+
assert.equal((await fetch(`${base}/nope`)).status, 404)
161+
})
162+
163+
it("answers a request without tools with plain JSON", async () => {
164+
const body = await (await complete({ messages: [{ role: "user", content: "hi" }] })).json()
165+
166+
assert.equal(body.object, "chat.completion")
167+
assert.match(body.choices[0].message.content, /Summary/)
168+
})
169+
170+
it("streams a scripted tool call, then attempt_completion once --max-requests is exceeded", async () => {
171+
const request = { stream: true, messages: [{ role: "user", content: "go" }], tools: [{ type: "function", function: { name: "x" } }] }
172+
173+
const first = await (await complete(request)).text()
174+
assert.match(first, /"tool_calls"/)
175+
assert.ok(first.trimEnd().endsWith("data: [DONE]"))
176+
assert.doesNotMatch(first, /attempt_completion/)
177+
178+
const second = await (await complete(request)).text()
179+
assert.match(second, /attempt_completion/)
180+
assert.ok(second.trimEnd().endsWith("data: [DONE]"))
181+
})
182+
})
183+
})

‎scripts/gray-screen/analyze-session.mjs‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -101,7 +101,19 @@ console.log(`storage: ${storage}`)
101101
console.log(`tasks: ${ids.length} (excluded ${sizes.length - real.length} generated [LOAD TEST] tasks${includeGenerated ? " - included" : ""}); analyzed set: ${real.length}`)
102102
console.log(`ui_messages.json size: p50=${mb(pct(0.5))}MB p90=${mb(pct(0.9))}MB p99=${mb(pct(0.99))}MB max=${mb(sorted.at(-1) ?? 0)}MB; over 10MB: ${sorted.filter((s) => s > 10 * 1048576).length}, over 5MB: ${sorted.filter((s) => s > 5 * 1048576).length}`)
103103

104-
const results = real.sort((a, b) => b.size - a.size).slice(0, top).map((s) => analyze(s.id))
104+
let unreadable = 0
105+
const results = real
106+
.sort((a, b) => b.size - a.size)
107+
.slice(0, top)
108+
.flatMap((s) => {
109+
try {
110+
return [analyze(s.id)]
111+
} catch {
112+
unreadable++
113+
return []
114+
}
115+
})
116+
if (unreadable) console.log(`skipped ${unreadable} task(s) whose ui_messages.json could not be parsed`)
105117
console.log(`\nTop ${results.length} tasks by ui_messages.json size (sizes only, no content):`)
106118
console.log("size(MB) msgs toolTxt(MB) editTools topField(share) 2byte% imgMB maxBatchRun(bytes MB) task")
107119
for (const r of results) {

‎scripts/gray-screen/lib.mjs‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,26 @@ export function validateRelativeDir(dir) {
3838
return dir
3939
}
4040

41+
const BUILD_MODES = ["production", "development"]
42+
43+
/**
44+
* The temp directory a stress-build of the given Vite mode is written to (Vite empties it first, so it must never
45+
* be derived from unchecked input). Refuses modes other than the documented ones and a symlinked output path.
46+
*/
47+
export function resolveBuildDir(mode, tmpRoot = "/tmp") {
48+
if (!BUILD_MODES.includes(mode)) throw new Error(`Invalid --build-mode "${mode}": use ${BUILD_MODES.join(" or ")}.`)
49+
50+
const dir = path.join(tmpRoot, "zoo-webview-stress-build" + (mode === "production" ? "" : `-${mode}`))
51+
52+
try {
53+
if (fs.lstatSync(dir).isSymbolicLink()) throw new Error(`Refusing to build into a symlink: ${dir}`)
54+
} catch (error) {
55+
if (error?.code !== "ENOENT") throw error
56+
}
57+
58+
return dir
59+
}
60+
4161
/** Resolves a request path to a regular file inside `root` (after symlink resolution), or undefined. */
4262
export function resolveServedFile(root, rawPath) {
4363
try {

‎scripts/gray-screen/webview-render-stress.mjs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ import http from "node:http"
2828
import { createRequire } from "node:module"
2929
import path from "node:path"
3030
import { fileURLToPath } from "node:url"
31-
import { resolveServedFile } from "./lib.mjs"
31+
import { resolveBuildDir, resolveServedFile } from "./lib.mjs"
3232

3333
const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..")
3434
const webviewDir = path.join(root, "webview-ui")
@@ -61,7 +61,7 @@ const cfg = {
6161
const sleep = (ms) => new Promise((r) => setTimeout(r, ms))
6262

6363
const buildMode = args["build-mode"] ?? "production"
64-
const buildDir = "/tmp/zoo-webview-stress-build" + (buildMode === "production" ? "" : `-${buildMode}`)
64+
const buildDir = resolveBuildDir(buildMode)
6565

6666
function run(cmd, cmdArgs, opts) {
6767
return new Promise((resolve, reject) => {

0 commit comments

Comments
 (0)