Skip to content

Commit 7bf8d8c

Browse files
authored
fix(remote): materialize test suite artifacts against a remote daemon (#2272)
* fix(remote): materialize test suite artifacts against a remote daemon (#2246) `agent-device test` crashed with ENOENT against a remote daemon because the scheduler resolved `--artifacts-dir` against the caller's `cwd`, sent over the wire, on the daemon's own filesystem. Mirrors #1802's read-side fix for the same command: the CLI now redirects `--artifacts-dir` to a temp directory the daemon owns before the suite runs, and the daemon rewrites every artifact path in its response back to the caller-local root and registers the suite directory as one downloadable artifact through the existing screenshot/record transport, extended here to also support directories via the codebase's existing safe archive extractor (the archive comes from a remote daemon, a different trust domain, so a raw `tar` invocation was not enough). * fix(remote): publish test artifacts atomically * perf(cli): keep artifact downloads lazy
1 parent 309a536 commit 7bf8d8c

15 files changed

Lines changed: 1042 additions & 124 deletions

‎packages/contracts/src/replay.ts‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -141,6 +141,13 @@ export type ReplaySuiteResult = {
141141
durationMs: number;
142142
failures: ReplaySuiteTestFailed[];
143143
tests: ReplaySuiteTestResult[];
144+
/**
145+
* The suite's own artifacts root (the parent of every test's `artifactsDir`), as resolved on
146+
* the host that ran the suite. Absent when the suite produced no attempt (e.g. every source
147+
* was filtered out). #2246: a remote daemon rewrites this to the caller-local path once the
148+
* directory has been transferred back, so it always names a path the caller can open.
149+
*/
150+
artifactsDir?: string;
144151
snapshotDiagnostics?: SnapshotDiagnosticsSummary;
145152
};
146153

‎packages/kernel/src/contracts.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -114,7 +114,8 @@ export type DaemonArtifactKnownType =
114114
| 'screen-recording'
115115
| 'screen-recording-chunk'
116116
| 'screen-recording-telemetry'
117-
| 'trace-log';
117+
| 'trace-log'
118+
| 'test-artifacts';
118119

119120
export type DaemonArtifactType = DaemonArtifactKnownType | (string & {});
120121

‎packages/replay-test/src/internal/__tests__/session-test-artifacts.test.ts‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,11 +4,27 @@ import fs from 'node:fs';
44
import os from 'node:os';
55
import path from 'node:path';
66
import {
7+
DEFAULT_TEST_ARTIFACTS_ROOT,
78
materializeReplayTestAttemptArtifacts,
89
prepareReplayTestAttemptArtifacts,
10+
resolveReplayTestArtifactsDir,
911
} from '../session-test-artifacts.ts';
1012
import type { ReplayTestAttemptOutcome } from '../session-test-types.ts';
1113

14+
test('resolveReplayTestArtifactsDir falls back to the default root when artifactsDir is omitted', () => {
15+
const dir = resolveReplayTestArtifactsDir({ cwd: '/repo', suiteInvocationId: 'abc123' });
16+
assert.equal(dir, path.resolve('/repo', DEFAULT_TEST_ARTIFACTS_ROOT, 'abc123'));
17+
});
18+
19+
test('resolveReplayTestArtifactsDir resolves an explicit relative artifactsDir against cwd', () => {
20+
const dir = resolveReplayTestArtifactsDir({
21+
artifactsDir: 'remote-device-artifacts/ad-test',
22+
cwd: '/repo',
23+
suiteInvocationId: 'abc123',
24+
});
25+
assert.equal(dir, path.resolve('/repo', 'remote-device-artifacts/ad-test', 'abc123'));
26+
});
27+
1228
// Building outcomes from a DaemonResponse is the adapter's job and is pinned on that side; a
1329
// package test states the neutral outcome directly (#1478 P3b).
1430
const passedOutcome = (

‎packages/replay-test/src/internal/session-test-artifacts.ts‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,15 @@ import path from 'node:path';
33
import { trimEdgeDashes } from '@agent-device/kernel/collections';
44
import type { ReplayTestAttemptOutcome } from '@agent-device/replay-test';
55

6-
const DEFAULT_TEST_ARTIFACTS_ROOT = '.agent-device/test-artifacts';
6+
/**
7+
* `test`'s default artifacts root when `--artifacts-dir` is not given. `src/remote/daemon-artifacts.ts`
8+
* keeps its own copy of this literal (to redirect a remote request without resolving anything
9+
* against the daemon's `cwd`, #2246): pulling it from `@agent-device/contracts` instead grew this
10+
* package's eager import closure by 3 modules for one string, past its pinned budget
11+
* (`scripts/__tests__/eager-closure-budgets.test.ts`) — not worth it for a value that changes
12+
* only if this line does.
13+
*/
14+
export const DEFAULT_TEST_ARTIFACTS_ROOT = '.agent-device/test-artifacts';
715

816
export function resolveReplayTestArtifactsDir(params: {
917
artifactsDir?: string;

‎packages/replay-test/src/internal/session-test.ts‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -124,7 +124,12 @@ export async function runReplayTestSuite(
124124
);
125125
}
126126

127-
const data = summarizeReplayTestResults(plan.total, results, Date.now() - suiteStartedAt);
127+
const data = summarizeReplayTestResults(
128+
plan.total,
129+
results,
130+
Date.now() - suiteStartedAt,
131+
plan.suiteArtifactsDir,
132+
);
128133
return { status: 'completed', data };
129134
} catch (error) {
130135
const appErr = asAppError(error);
@@ -467,6 +472,7 @@ function summarizeReplayTestResults(
467472
total: number,
468473
results: ReplaySuiteTestResult[],
469474
durationMs: number,
475+
artifactsDir: string,
470476
): ReplaySuiteResult {
471477
const passed = results.filter((result) => result.status === 'passed').length;
472478
const failedResults = results.filter(
@@ -488,6 +494,7 @@ function summarizeReplayTestResults(
488494
durationMs,
489495
failures: failedResults,
490496
tests: results,
497+
artifactsDir,
491498
...(snapshotDiagnostics ? { snapshotDiagnostics } : {}),
492499
};
493500
}
Lines changed: 172 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,172 @@
1+
/**
2+
* #2246: once a remote daemon writes suite artifacts under a temp directory it owns (the client
3+
* redirects `--artifacts-dir` there — see `daemon-artifacts-test-command.test.ts`), the response
4+
* this handler builds must point the caller back at the REAL local root the client will download
5+
* that directory into (`req.meta.clientArtifactPaths.artifactsDir`), and must register the
6+
* directory as one downloadable artifact so the client's existing artifact transport can pull it.
7+
* A local daemon never sets that hint, so the response must come back byte-for-byte unchanged.
8+
*
9+
* `daemonRoot` here is deliberately two levels (`<tempRoot>/<suiteInvocationId>`), matching what
10+
* `resolveReplayTestArtifactsDir` actually produces: the redirected temp ROOT the client sent
11+
* joined with the daemon-generated invocation id. The caller-local mirror of that is
12+
* `clientRoot/<suiteInvocationId>` — NOT bare `clientRoot`, which is only the root the client
13+
* resolved from `--artifacts-dir` *before* the suite ran and could not yet know the invocation id.
14+
*/
15+
import assert from 'node:assert/strict';
16+
import fs from 'node:fs';
17+
import path from 'node:path';
18+
import { test } from 'vitest';
19+
import type { ReplaySuiteResult } from '@agent-device/contracts/replay';
20+
import type { DaemonRequest } from '../../../types.ts';
21+
import { attachRemoteReplayTestArtifacts } from '../test-command.ts';
22+
import { mkdtempForTestSync } from '../../../../__tests__/test-utils/tmp-dir.ts';
23+
24+
const SUITE_INVOCATION_ID = 'cd5f9c01feec8d70';
25+
26+
function suiteResult(daemonRoot: string): ReplaySuiteResult {
27+
return {
28+
total: 2,
29+
executed: 2,
30+
passed: 1,
31+
failed: 1,
32+
skipped: 1,
33+
notRun: 0,
34+
durationMs: 10,
35+
tests: [
36+
{
37+
file: path.join(daemonRoot, '..', '01-open.ad'),
38+
session: 'default:test:1',
39+
status: 'passed',
40+
durationMs: 5,
41+
attempts: 1,
42+
artifactsDir: path.join(daemonRoot, 'qa-flows__open.ad', 'attempt-1'),
43+
replayed: 1,
44+
healed: 0,
45+
},
46+
{
47+
file: path.join(daemonRoot, '..', '02-checkout.ad'),
48+
session: 'default:test:2',
49+
status: 'failed',
50+
durationMs: 8,
51+
attempts: 1,
52+
artifactsDir: path.join(daemonRoot, 'qa-flows__checkout.ad', 'attempt-1'),
53+
error: { code: 'COMMAND_FAILED', message: 'boom' },
54+
},
55+
{
56+
file: 'skipped.ad',
57+
status: 'skipped',
58+
durationMs: 0,
59+
reason: 'skipped-by-filter',
60+
message: 'platform mismatch',
61+
},
62+
],
63+
// The failed test above is intentionally the SAME object reference here, mirroring
64+
// `summarizeReplayTestResults` (`failures: results.filter(...)`, `tests: results`) — a fix
65+
// that only rewrites `tests` and forgets `failures` would leave this one unrewritten.
66+
get failures() {
67+
return this.tests.filter(
68+
(t): t is Extract<ReplaySuiteResult['tests'][number], { status: 'failed' }> =>
69+
t.status === 'failed',
70+
);
71+
},
72+
artifactsDir: daemonRoot,
73+
};
74+
}
75+
76+
function req(clientArtifactsRoot: string | undefined): DaemonRequest {
77+
return {
78+
token: 't',
79+
session: 'default',
80+
command: 'test',
81+
positionals: [],
82+
meta: clientArtifactsRoot ? { clientArtifactPaths: { artifactsDir: clientArtifactsRoot } } : {},
83+
};
84+
}
85+
86+
test('rewrites the suite, every test, and every failure to the caller-local root, and registers the directory for download', () => {
87+
const tempRoot = mkdtempForTestSync('agent-device-remote-test-artifacts-daemon-');
88+
const daemonRoot = path.join(tempRoot, SUITE_INVOCATION_ID);
89+
fs.mkdirSync(path.join(daemonRoot, 'qa-flows__open.ad', 'attempt-1'), { recursive: true });
90+
const clientRoot = '/Users/ci/work/installer-app/remote-device-artifacts/ad-test';
91+
const clientSuiteRoot = path.join(clientRoot, SUITE_INVOCATION_ID);
92+
93+
const data = attachRemoteReplayTestArtifacts(suiteResult(daemonRoot), req(clientRoot));
94+
95+
assert.equal(data.artifactsDir, clientSuiteRoot);
96+
const tests = data.tests as ReplaySuiteResult['tests'];
97+
assert.equal(
98+
(tests[0] as { artifactsDir?: string }).artifactsDir,
99+
path.join(clientSuiteRoot, 'qa-flows__open.ad', 'attempt-1'),
100+
);
101+
assert.equal(
102+
(tests[1] as { artifactsDir?: string }).artifactsDir,
103+
path.join(clientSuiteRoot, 'qa-flows__checkout.ad', 'attempt-1'),
104+
);
105+
// A skipped test never had an artifactsDir; it must not gain one.
106+
const skippedTest = tests[2];
107+
assert.ok(skippedTest);
108+
assert.equal('artifactsDir' in skippedTest, false);
109+
110+
const failures = data.failures as ReplaySuiteResult['failures'];
111+
assert.equal(failures.length, 1);
112+
assert.equal(
113+
(failures[0] as { artifactsDir?: string }).artifactsDir,
114+
path.join(clientSuiteRoot, 'qa-flows__checkout.ad', 'attempt-1'),
115+
);
116+
117+
assert.deepEqual(data.artifacts, [
118+
{
119+
field: 'artifactsDir',
120+
artifactType: 'test-artifacts',
121+
path: daemonRoot,
122+
// The download destination is the ROOT the client redirected to, not `clientSuiteRoot`:
123+
// extracting the archive reproduces the invocation-id segment on its own.
124+
localPath: clientRoot,
125+
fileName: SUITE_INVOCATION_ID,
126+
},
127+
]);
128+
});
129+
130+
test('a local daemon (no clientArtifactPaths hint) returns the suite result unchanged', () => {
131+
const tempRoot = mkdtempForTestSync('agent-device-remote-test-artifacts-local-');
132+
const daemonRoot = path.join(tempRoot, SUITE_INVOCATION_ID);
133+
fs.mkdirSync(path.join(daemonRoot, 'qa-flows__open.ad', 'attempt-1'), { recursive: true });
134+
const result = suiteResult(daemonRoot);
135+
136+
const data = attachRemoteReplayTestArtifacts(result, req(undefined));
137+
138+
assert.equal(data, result);
139+
});
140+
141+
test('a suite directory that was never created (all sources skipped) still reports the caller-local root, but registers no download', () => {
142+
const tempRoot = mkdtempForTestSync('agent-device-remote-test-artifacts-empty-');
143+
const daemonRoot = path.join(tempRoot, SUITE_INVOCATION_ID);
144+
const clientRoot = '/Users/ci/work/installer-app/remote-device-artifacts/ad-test';
145+
const result: ReplaySuiteResult = {
146+
total: 1,
147+
executed: 0,
148+
passed: 0,
149+
failed: 0,
150+
skipped: 1,
151+
notRun: 0,
152+
durationMs: 1,
153+
failures: [],
154+
artifactsDir: daemonRoot,
155+
tests: [
156+
{
157+
file: 'skipped.ad',
158+
status: 'skipped',
159+
durationMs: 0,
160+
reason: 'skipped-by-filter',
161+
message: 'x',
162+
},
163+
],
164+
};
165+
166+
const data = attachRemoteReplayTestArtifacts(result, req(clientRoot));
167+
168+
// Nothing to download, but the reported root is still the caller-local one the suite WOULD
169+
// have used — not the unreachable daemon-local temp path.
170+
assert.equal(data.artifactsDir, path.join(clientRoot, SUITE_INVOCATION_ID));
171+
assert.equal(data.artifacts, undefined);
172+
});

‎src/daemon/replay/internal/test-command.ts‎

Lines changed: 84 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,12 @@
11
/** Runs the replay-test scheduler and its nested replay attempts. */
22

3+
import fs from 'node:fs';
4+
import path from 'node:path';
35
import type { CommandFlags } from '@agent-device/contracts/command';
4-
import type { ReplayScriptSourceBundle } from '@agent-device/contracts/replay';
6+
import type { ReplaySuiteResult, ReplayScriptSourceBundle } from '@agent-device/contracts/replay';
57
import { REPLAY_SCRIPT_SOURCE_REQUIRED_MESSAGE } from '../../replay-script-source.ts';
68
import type { ReplayScriptMetadata } from '@agent-device/ad-script';
7-
import type { DaemonRequest, DaemonResponse } from '../../types.ts';
9+
import type { DaemonRequest, DaemonResponse, DaemonResponseData } from '../../types.ts';
810
import { expandSessionPath } from '../../session-paths.ts';
911
import type { ReplayTestCommand } from './command-types.ts';
1012
import {
@@ -279,10 +281,89 @@ export async function runReplayTestCommand(command: ReplayTestCommand): Promise<
279281
cleanupSession,
280282
});
281283
return outcome.status === 'completed'
282-
? { ok: true, data: outcome.data }
284+
? { ok: true, data: attachRemoteReplayTestArtifacts(outcome.data, req) }
283285
: errorResponse(outcome.error.code, outcome.error.message);
284286
}
285287

288+
/**
289+
* #2246: against a remote daemon, the client redirects `--artifacts-dir` to a temp directory on
290+
* THIS host before the suite runs (`prepareRemoteRequestArtifacts` in
291+
* `src/remote/daemon-artifacts.ts`) — the caller's real `cwd` names no path here, so the
292+
* scheduler above must never resolve suite artifacts against it. `req.meta.clientArtifactPaths`
293+
* is how that client tells the daemon the real caller-local ROOT (e.g. `--artifacts-dir`, or its
294+
* default) it will pull the directory back into once the request returns — that root is resolved
295+
* client-side *before* the suite runs, so it never contains the suite's own invocation id.
296+
*
297+
* `data.artifactsDir` (the daemon-local suite directory) is always `<redirected root>/<suite
298+
* invocation id>`: `createDirectoryArchive` (`src/daemon/artifact-tracking.ts`) tars it as
299+
* `-C dirname(daemonRoot) -- basename(daemonRoot)`, so extracting that archive under the
300+
* caller-local root reproduces the SAME invocation-id segment there. The caller-facing root is
301+
* therefore `path.join(clientRoot, basename(daemonRoot))`, not `clientRoot` itself — every path
302+
* this rewrites must land under that, matching where the client will actually put the files.
303+
*
304+
* Rewriting every artifact path in the response to that root, and registering the directory
305+
* itself as one downloadable artifact (the same generic `data.artifacts` -> `trackArtifact`
306+
* mechanism `screenshot`/`record` already use), are host concerns: the scheduler only ever
307+
* reports paths on its own filesystem.
308+
*
309+
* A local daemon never sets `clientArtifactPaths` (the client only redirects for a remote one),
310+
* so this is a no-op there and the response is returned unchanged.
311+
*/
312+
export function attachRemoteReplayTestArtifacts(
313+
data: ReplaySuiteResult,
314+
req: DaemonRequest,
315+
): DaemonResponseData {
316+
const clientRoot = req.meta?.clientArtifactPaths?.artifactsDir;
317+
const daemonRoot = data.artifactsDir;
318+
if (!clientRoot || !daemonRoot) return data;
319+
320+
// The directory the client will actually extract the download into, on ITS filesystem —
321+
// `clientRoot` plus the same invocation-id segment `daemonRoot` ends in (see doc comment).
322+
const clientSuiteRoot = path.join(clientRoot, path.basename(daemonRoot));
323+
const remapArtifactsDir = (candidate: string): string =>
324+
candidate === daemonRoot || candidate.startsWith(daemonRoot + path.sep)
325+
? clientSuiteRoot + candidate.slice(daemonRoot.length)
326+
: candidate;
327+
// `tests` and `failures` both come from the scheduler's own `results` array
328+
// (`summarizeReplayTestResults`) and share object references for every failed entry — each
329+
// array is rewritten independently so both, not just whichever a caller happens to read, carry
330+
// the caller-local path.
331+
const rewritten: DaemonResponseData = {
332+
...data,
333+
artifactsDir: clientSuiteRoot,
334+
tests: data.tests.map((test) =>
335+
'artifactsDir' in test && test.artifactsDir
336+
? { ...test, artifactsDir: remapArtifactsDir(test.artifactsDir) }
337+
: test,
338+
),
339+
failures: data.failures.map((failure) =>
340+
failure.artifactsDir
341+
? { ...failure, artifactsDir: remapArtifactsDir(failure.artifactsDir) }
342+
: failure,
343+
),
344+
};
345+
346+
// No attempt ever ran (e.g. every source was filtered out), so the directory the client would
347+
// try to download was never created — the rewritten paths above are still correct (they name
348+
// where the suite WOULD have put its output), just nothing to register a download for.
349+
if (!fs.existsSync(daemonRoot)) return rewritten;
350+
351+
return {
352+
...rewritten,
353+
artifacts: [
354+
{
355+
field: 'artifactsDir',
356+
artifactType: 'test-artifacts',
357+
path: daemonRoot,
358+
// The download destination is the ROOT, not `clientSuiteRoot`: the archive's own
359+
// top-level entry already supplies the invocation-id segment on extraction.
360+
localPath: clientRoot,
361+
fileName: path.basename(daemonRoot),
362+
},
363+
],
364+
};
365+
}
366+
286367
/**
287368
* Translates a daemon `test` request into the scheduler's neutral request (#1478 P3b).
288369
*

0 commit comments

Comments
 (0)