Skip to content

Commit 85a16e7

Browse files
committed
test(e2e): verify persistence across VS Code restart
1 parent 09e8232 commit 85a16e7

8 files changed

Lines changed: 442 additions & 17 deletions

File tree

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
{
2+
"fixtures": [
3+
{
4+
"match": {
5+
"userMessage": "RESTART_PERSISTENCE_SMOKE"
6+
},
7+
"response": {
8+
"toolCalls": [
9+
{
10+
"name": "attempt_completion",
11+
"arguments": "{\"result\":\"RESTART_PERSISTENCE_MARKER\"}",
12+
"id": "call_restart_persistence_done"
13+
}
14+
]
15+
}
16+
}
17+
]
18+
}
Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
import * as path from "path"
2+
import * as fs from "fs/promises"
3+
4+
import { createWriteStream } from "fs"
5+
6+
export const PHASE_RESULT_VERSION = 1 as const
7+
8+
export type RestartPhase = "create" | "verify"
9+
export type PhaseStatus = "passed" | "failed"
10+
11+
export type PhaseError = {
12+
message: string
13+
stack?: string
14+
}
15+
16+
export type PhaseResult = {
17+
version: typeof PHASE_RESULT_VERSION
18+
phase: RestartPhase
19+
status: PhaseStatus
20+
values?: Record<string, string>
21+
error?: PhaseError
22+
}
23+
24+
const phaseResultNames: Record<RestartPhase, string> = {
25+
create: "01-create.json",
26+
verify: "02-verify.json",
27+
}
28+
29+
export function getPhaseResultPath(resultsDir: string, phase: RestartPhase): string {
30+
const resolvedResultsDir = path.resolve(resultsDir)
31+
const resultPath = path.resolve(resolvedResultsDir, phaseResultNames[phase])
32+
if (path.dirname(resultPath) !== resolvedResultsDir) {
33+
throw new Error(`Phase result path escaped the results directory: ${phase}`)
34+
}
35+
return resultPath
36+
}
37+
38+
export function serializePhaseError(error: unknown): PhaseError {
39+
if (error instanceof Error) {
40+
return {
41+
message: error.message.slice(0, 2_000),
42+
...(error.stack && { stack: error.stack.slice(0, 4_000) }),
43+
}
44+
}
45+
46+
return { message: String(error).slice(0, 2_000) }
47+
}
48+
49+
function isRecord(value: unknown): value is Record<string, unknown> {
50+
return typeof value === "object" && value !== null
51+
}
52+
53+
export function validatePhaseResult(value: unknown): asserts value is PhaseResult {
54+
if (!isRecord(value) || value.version !== PHASE_RESULT_VERSION) {
55+
throw new Error("Invalid phase result version")
56+
}
57+
if (value.phase !== "create" && value.phase !== "verify") {
58+
throw new Error("Invalid phase result phase")
59+
}
60+
if (value.status !== "passed" && value.status !== "failed") {
61+
throw new Error("Invalid phase result status")
62+
}
63+
if (value.values !== undefined) {
64+
if (!isRecord(value.values) || Object.values(value.values).some((entry) => typeof entry !== "string")) {
65+
throw new Error("Phase result values must be string-valued")
66+
}
67+
}
68+
if (value.error !== undefined) {
69+
if (!isRecord(value.error) || typeof value.error.message !== "string") {
70+
throw new Error("Invalid phase result error")
71+
}
72+
if (value.error.stack !== undefined && typeof value.error.stack !== "string") {
73+
throw new Error("Invalid phase result error stack")
74+
}
75+
}
76+
}
77+
78+
export async function writePhaseResult(resultsDir: string, result: PhaseResult): Promise<void> {
79+
validatePhaseResult(result)
80+
const targetPath = getPhaseResultPath(resultsDir, result.phase)
81+
const temporaryPath = `${targetPath}.${process.pid}.${Date.now()}.tmp`
82+
try {
83+
await writeJsonAtomically(temporaryPath, result)
84+
await fs.rename(temporaryPath, targetPath)
85+
} finally {
86+
await fs.rm(temporaryPath, { force: true })
87+
}
88+
}
89+
90+
async function writeJsonAtomically(filePath: string, value: PhaseResult): Promise<void> {
91+
await fs.mkdir(path.dirname(filePath), { recursive: true })
92+
await new Promise<void>((resolve, reject) => {
93+
const stream = createWriteStream(filePath, { encoding: "utf8" })
94+
stream.once("error", reject)
95+
stream.once("finish", resolve)
96+
stream.end(JSON.stringify(value))
97+
})
98+
}
99+
100+
export async function readPhaseResult(resultsDir: string, phase: RestartPhase): Promise<PhaseResult> {
101+
const result = JSON.parse(await fs.readFile(getPhaseResultPath(resultsDir, phase), "utf8")) as unknown
102+
validatePhaseResult(result)
103+
if (result.phase !== phase) {
104+
throw new Error(`Phase result does not match requested phase: ${phase}`)
105+
}
106+
return result
107+
}
Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
import * as os from "os"
2+
import * as path from "path"
3+
import * as fs from "fs/promises"
4+
5+
const SCENARIO_ROOT_PREFIX = "roo-vscode-e2e-restart-"
6+
7+
export type ScenarioWorkspace = {
8+
root: string
9+
workspace: string
10+
userData: string
11+
extensions: string
12+
results: string
13+
}
14+
15+
function childPath(root: string, name: string): string {
16+
const resolvedRoot = path.resolve(root)
17+
const resolvedChild = path.resolve(resolvedRoot, name)
18+
if (path.dirname(resolvedChild) !== resolvedRoot) {
19+
throw new Error(`Scenario path escaped its root: ${name}`)
20+
}
21+
return resolvedChild
22+
}
23+
24+
export async function createScenarioWorkspace(): Promise<ScenarioWorkspace> {
25+
const root = await fs.mkdtemp(path.join(os.tmpdir(), SCENARIO_ROOT_PREFIX))
26+
const scenarioWorkspace: ScenarioWorkspace = {
27+
root,
28+
workspace: childPath(root, "workspace"),
29+
userData: childPath(root, "user-data"),
30+
extensions: childPath(root, "extensions"),
31+
results: childPath(root, "results"),
32+
}
33+
34+
await Promise.all(
35+
[
36+
scenarioPath(scenarioWorkspace, "workspace"),
37+
scenarioPath(scenarioWorkspace, "userData"),
38+
scenarioPath(scenarioWorkspace, "extensions"),
39+
scenarioPath(scenarioWorkspace, "results"),
40+
].map((directory) => fs.mkdir(directory, { recursive: true })),
41+
)
42+
43+
return scenarioWorkspace
44+
}
45+
46+
function scenarioPath(scenarioWorkspace: ScenarioWorkspace, key: "workspace" | "userData" | "extensions" | "results") {
47+
return scenarioWorkspace[key]
48+
}
49+
50+
export async function removeScenarioWorkspace(scenarioWorkspace: ScenarioWorkspace): Promise<void> {
51+
const root = path.resolve(scenarioWorkspace.root)
52+
const tempRoot = path.resolve(os.tmpdir())
53+
if (path.dirname(root) !== tempRoot || !path.basename(root).startsWith(SCENARIO_ROOT_PREFIX)) {
54+
throw new Error(`Refusing to remove an unowned scenario root: ${scenarioWorkspace.root}`)
55+
}
56+
57+
await fs.rm(root, { recursive: true, force: true })
58+
}
Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
import { spawn } from "child_process"
2+
import * as path from "path"
3+
4+
import { readPhaseResult, type RestartPhase, type PhaseResult } from "./phaseProtocol"
5+
import type { ScenarioWorkspace } from "./scenarioWorkspace"
6+
7+
export type RestartCoordinatorOptions = {
8+
vscodeExecutablePath: string
9+
extensionDevelopmentPath: string
10+
extensionTestsPath: string
11+
scenario: string
12+
workspace: ScenarioWorkspace
13+
environment: NodeJS.ProcessEnv
14+
expectedExitPolicies: readonly ExpectedExitPolicy[]
15+
}
16+
17+
export type ExpectedExitPolicy = {
18+
phase: RestartPhase
19+
termination: "graceful-quit"
20+
code: number
21+
signal: NodeJS.Signals | null
22+
}
23+
24+
function phaseEnvironment(options: RestartCoordinatorOptions, phase: RestartPhase): NodeJS.ProcessEnv {
25+
return {
26+
...options.environment,
27+
E2E_PHASE: phase,
28+
E2E_SCENARIO: options.scenario,
29+
E2E_RESULTS_DIR: options.workspace.results,
30+
}
31+
}
32+
33+
function phaseArguments(options: RestartCoordinatorOptions): string[] {
34+
return [
35+
options.workspace.workspace,
36+
`--user-data-dir=${options.workspace.userData}`,
37+
`--extensions-dir=${options.workspace.extensions}`,
38+
"--no-sandbox",
39+
"--disable-gpu-sandbox",
40+
"--disable-updates",
41+
"--skip-welcome",
42+
"--skip-release-notes",
43+
"--disable-workspace-trust",
44+
`--extensionTestsPath=${options.extensionTestsPath}`,
45+
`--extensionDevelopmentPath=${options.extensionDevelopmentPath}`,
46+
]
47+
}
48+
49+
async function runPhase(options: RestartCoordinatorOptions, phase: RestartPhase): Promise<PhaseResult> {
50+
const args = phaseArguments(options)
51+
console.log(`[restart:${phase}] spawning VS Code: ${path.basename(options.vscodeExecutablePath)}`)
52+
53+
const child = spawn(options.vscodeExecutablePath, args, {
54+
env: phaseEnvironment(options, phase),
55+
shell: process.platform === "win32",
56+
stdio: ["ignore", "pipe", "pipe"],
57+
})
58+
59+
child.stdout?.on("data", (chunk: Buffer) => process.stdout.write(`[restart:${phase}] ${chunk}`))
60+
child.stderr?.on("data", (chunk: Buffer) => process.stderr.write(`[restart:${phase}] ${chunk}`))
61+
62+
const exit = await new Promise<{ code: number | null; signal: NodeJS.Signals | null }>((resolve, reject) => {
63+
child.once("error", reject)
64+
child.once("close", (code, signal) => resolve({ code, signal }))
65+
})
66+
67+
const result = await readPhaseResult(options.workspace.results, phase)
68+
if (result.status !== "passed") {
69+
throw new Error(`[restart:${phase}] phase result was not passed: ${result.error?.message ?? "unknown failure"}`)
70+
}
71+
72+
const expectedExit = options.expectedExitPolicies.find((policy) => policy.phase === phase)
73+
const isExpectedNonzeroExit =
74+
exit.code !== 0 &&
75+
expectedExit !== undefined &&
76+
expectedExit.termination === "graceful-quit" &&
77+
exit.code === expectedExit.code &&
78+
exit.signal === expectedExit.signal
79+
const isSuccessfulExit = exit.code === 0 && exit.signal === null
80+
if (!isSuccessfulExit && !isExpectedNonzeroExit) {
81+
throw new Error(`[restart:${phase}] VS Code exited with ${exit.code ?? exit.signal}`)
82+
}
83+
84+
return result
85+
}
86+
87+
export async function runRestartScenario(options: RestartCoordinatorOptions): Promise<void> {
88+
await runPhase(options, "create")
89+
await runPhase(options, "verify")
90+
}

‎apps/vscode-e2e/src/runTest.ts‎

Lines changed: 47 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ import * as os from "os"
33
import * as fs from "fs/promises"
44
import { readFileSync } from "fs"
55

6-
import { runTests } from "@vscode/test-electron"
6+
import { downloadAndUnzipVSCode, runTests } from "@vscode/test-electron"
77
import { LLMock } from "@copilotkit/aimock"
88

99
import { addApplyDiffResultFixtures } from "./fixtures/apply-diff"
@@ -21,6 +21,8 @@ import { addSearchFilesResultFixtures } from "./fixtures/search-files"
2121
import { addSubtaskFixtures } from "./fixtures/subtasks"
2222
import { addUseMcpToolResultFixtures } from "./fixtures/use-mcp-tool"
2323
import { addWriteToFileResultFixtures } from "./fixtures/write-to-file"
24+
import { createScenarioWorkspace, removeScenarioWorkspace } from "./restart/scenarioWorkspace"
25+
import { runRestartScenario } from "./restart/vscodeCoordinator"
2426

2527
function getCliFlagValue(flag: string) {
2628
return process.argv.find((arg, index) => process.argv[index - 1] === flag)
@@ -43,13 +45,18 @@ function isBedrockTargetedRun(testFile?: string, testGrep?: string) {
4345
return testGrep?.toLowerCase().includes("bedrock") ?? false
4446
}
4547

48+
function isRestartPersistenceTargetedRun(testFile?: string): boolean {
49+
return testFile?.toLowerCase().includes("restart-persistence") ?? false
50+
}
51+
4652
async function main() {
4753
const isRecord = process.env.AIMOCK_RECORD === "true"
4854
const testGrep = getCliFlagValue("--grep") || process.env.TEST_GREP
4955
const testFile = getCliFlagValue("--file") || process.env.TEST_FILE
5056
const isDeepSeekTest = isDeepSeekTargetedRun(testFile, testGrep)
5157
const isGeminiTest = testFile?.toLowerCase().includes("gemini.test") ?? false
5258
const isBedrockTest = isBedrockTargetedRun(testFile, testGrep)
59+
const isRestartPersistenceTest = isRestartPersistenceTargetedRun(testFile)
5360

5461
if (isRecord && isDeepSeekTest && !process.env.DEEPSEEK_API_KEY) {
5562
throw new Error("AIMOCK_RECORD=true requires DEEPSEEK_API_KEY to record DeepSeek fixtures")
@@ -83,11 +90,14 @@ async function main() {
8390
const extensionTestsPath = path.resolve(__dirname, "./suite/index")
8491

8592
let testWorkspace: string | undefined
93+
let scenarioWorkspace: Awaited<ReturnType<typeof createScenarioWorkspace>> | undefined
8694

8795
try {
88-
// Create a temporary workspace folder for tests before installing fixtures that
89-
// need workspace-specific paths.
90-
testWorkspace = await fs.mkdtemp(path.join(os.tmpdir(), "roo-test-workspace-"))
96+
// Create a temporary workspace folder for regular tests. Restart scenarios own
97+
// all of their paths under the dedicated scenario root below.
98+
if (!isRestartPersistenceTest) {
99+
testWorkspace = await fs.mkdtemp(path.join(os.tmpdir(), "roo-test-workspace-"))
100+
}
91101

92102
if (useMock) {
93103
const fixturesDir = path.resolve(__dirname, "../fixtures")
@@ -173,20 +183,46 @@ async function main() {
173183
const pkg = JSON.parse(readFileSync(path.resolve(__dirname, "../package.json"), "utf-8"))
174184
const vscodeVersion = process.env.VSCODE_VERSION || pkg.devDependencies["@types/vscode"]
175185

176-
await runTests({
177-
extensionDevelopmentPath,
178-
extensionTestsPath,
179-
launchArgs: [testWorkspace],
180-
extensionTestsEnv,
181-
version: vscodeVersion,
182-
})
186+
if (isRestartPersistenceTest) {
187+
scenarioWorkspace = await createScenarioWorkspace()
188+
const vscodeExecutablePath = await downloadAndUnzipVSCode({
189+
version: vscodeVersion,
190+
extensionDevelopmentPath,
191+
})
192+
await runRestartScenario({
193+
vscodeExecutablePath,
194+
extensionDevelopmentPath,
195+
extensionTestsPath,
196+
scenario: "restart-persistence",
197+
workspace: scenarioWorkspace,
198+
environment: extensionTestsEnv,
199+
expectedExitPolicies: [
200+
{ phase: "create", termination: "graceful-quit", code: 1, signal: null },
201+
{ phase: "verify", termination: "graceful-quit", code: 1, signal: null },
202+
],
203+
})
204+
} else {
205+
if (!testWorkspace) {
206+
throw new Error("Regular E2E runs require a temporary test workspace")
207+
}
208+
await runTests({
209+
extensionDevelopmentPath,
210+
extensionTestsPath,
211+
launchArgs: [testWorkspace],
212+
extensionTestsEnv,
213+
version: vscodeVersion,
214+
})
215+
}
183216
} catch (error) {
184217
console.error("Failed to run tests", error)
185218
process.exitCode = 1
186219
} finally {
187220
if (testWorkspace) {
188221
await fs.rm(testWorkspace, { recursive: true, force: true })
189222
}
223+
if (scenarioWorkspace) {
224+
await removeScenarioWorkspace(scenarioWorkspace)
225+
}
190226
await mock?.stop()
191227
}
192228
}

‎apps/vscode-e2e/src/suite/index.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,7 @@ export async function run() {
7272
testFiles = await glob(`**/${specificFile}`, { cwd })
7373
console.log(`Running specific test file: ${specificFile}`)
7474
} else {
75-
testFiles = await glob("**/**.test.js", { cwd })
75+
testFiles = await glob("**/**.test.js", { cwd, ignore: "**/suite/restart-persistence.test.js" })
7676
}
7777

7878
if (testFiles.length === 0) {

0 commit comments

Comments
 (0)