Skip to content

Commit 6380023

Browse files
committed
🔒 Give a workflow Agent nothing but what its prompt renders
#302 settles that a workflow Agent receives no Workspace checkout, no read-only materialization of one, no Workspace or host path as cwd, no ACP `additionalDirectories`, and no component that registers a directory. These documents described the opposite in a dozen places: `<Agent.AddDir>` in four stages, a `worktree` prop threaded from the root into three of them, prompts telling agents to investigate a registered checkout, and an inventory listing a directory component as work still to do. All of it is gone. The stages take no `worktree`, the test stub is deleted, and the prompts say what is true — each agent reasons over what its prompt renders and says plainly what it cannot determine. `InstructionFiles` is the concrete case that survives: host-authored file effects produce exact repository-relative paths and contents, and that captured result reaches Discovery and both Planning prompts as data. General repository observation stays the bounded XMD request/result loop #302 and #369 owe, and this change invents no spelling for it. The boundary is now proved, not asserted. The root-provider stub records the whole Agent-facing surface — factory options, agent and session selection, prompt text and prompt options — and a root that reads real instruction files proves the paths and contents arrive while the absolute directory they were read from does not, along with no Workspace value, checkout path, registration input or `additionalDirectories` field. Injecting that directory trips the guard, and summarizing the instructions fails the arrival check. `<Agent.AddDir>` leaves the inventory rather than moving down it: it is excluded by architecture, not waiting on implementation. Four names remain owed — `<Expand>`, `<Git.Push>`, `<PullRequest>`, `<Issue>` — and #293's Worktree identity, replay and cleanup question moves from Open to Settled, where the shipped contract belongs.
1 parent 25596ea commit 6380023

15 files changed

Lines changed: 383 additions & 177 deletions

‎scripts/tests/adversarial-planning-workflow.test.ts‎

Lines changed: 217 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,8 @@
1010
*
1111
* Nothing about the document under test changes. The component resolves from
1212
* the workflow directory exactly as `xmd test` resolves it, including the
13-
* test-local `Agent.AddDir` stub, and the loop bound is the one the workflow
14-
* ships.
13+
* loop bound is the one the workflow ships. No stub stands in for a directory
14+
* component, because #302 gives a workflow Agent none.
1515
*
1616
* What `start.md` does with the returned pair — placing `<Implementation>`
1717
* behind the same gate and reporting exhaustion as awaiting direction inside
@@ -26,6 +26,7 @@ import { ensure, scoped } from "effection";
2626
import type { Operation, Stream } from "effection";
2727
import { InMemoryStream } from "@executablemd/durable-streams";
2828
import { ensureDir, rm, writeTextFile } from "@effectionx/fs";
29+
import { API, useHostFiles } from "@executablemd/runtime";
2930
import { randomUUID } from "node:crypto";
3031
import * as path from "node:path";
3132
import * as os from "node:os";
@@ -48,13 +49,16 @@ const COMPONENT_DIRS = [path.join(WORKFLOW, "tests"), WORKFLOW];
4849
const MAX_ITERATIONS = 5;
4950

5051
/** Which prompt a turn is, read from the text the document actually sent. */
51-
type Turn = "plan" | "verdict" | "assessment" | "revision" | "sentinel";
52+
type Turn = "discovery" | "plan" | "verdict" | "assessment" | "revision" | "sentinel";
5253

5354
function classify(content: string): Turn {
5455
if (content.includes("Revise the implementation plan using this review:")) {
5556
return "revision";
5657
}
57-
if (content.includes("Investigate the registered checkout.")) {
58+
if (content.includes("Produce a user-validated design handoff")) {
59+
return "discovery";
60+
}
61+
if (content.includes("amend the implementation theory against that material")) {
5862
return "plan";
5963
}
6064
if (content.includes("Review the plan against the handoff")) {
@@ -70,6 +74,42 @@ interface Call {
7074
readonly turn: Turn;
7175
readonly session: string;
7276
readonly content: string;
77+
/** Everything the prompt carried besides its text. */
78+
readonly options: Readonly<Record<string, unknown>>;
79+
readonly agent: string;
80+
}
81+
82+
/** The complete Agent-facing trace: what the factory, selections and prompts saw. */
83+
interface Trace {
84+
readonly calls: Call[];
85+
readonly factoryOptions: Record<string, unknown>[];
86+
readonly agentSelections: (string | undefined)[];
87+
readonly sessionSelections: (string | undefined)[];
88+
}
89+
90+
/** Every string an Agent could have observed, flattened for boundary checks. */
91+
function agentFacingStrings(trace: Trace): string[] {
92+
const out: string[] = [];
93+
const walk = (value: unknown) => {
94+
if (typeof value === "string") {
95+
out.push(value);
96+
} else if (Array.isArray(value)) {
97+
value.forEach(walk);
98+
} else if (value && typeof value === "object") {
99+
for (const [k, v] of Object.entries(value)) {
100+
out.push(k);
101+
walk(v);
102+
}
103+
}
104+
};
105+
for (const call of trace.calls) {
106+
out.push(call.content, call.session, call.agent);
107+
walk(call.options);
108+
}
109+
trace.factoryOptions.forEach(walk);
110+
trace.agentSelections.forEach((v) => v !== undefined && out.push(v));
111+
trace.sessionSelections.forEach((v) => v !== undefined && out.push(v));
112+
return out;
73113
}
74114

75115
/** `PromptOptions.session` is `string | Session`; the union narrows on its own. */
@@ -88,6 +128,8 @@ function sessionName(options: PromptOptions | undefined): string {
88128
*/
89129
function reply(turn: Turn, round: number): string {
90130
switch (turn) {
131+
case "discovery":
132+
return `HANDOFF-ROUND-${round} the route is /health.`;
91133
case "plan":
92134
return `PLAN-ROUND-${round}`;
93135
case "verdict":
@@ -111,25 +153,34 @@ function reply(turn: Turn, round: number): string {
111153
}
112154
}
113155

114-
function stubProvider(calls: Call[]): AgentProviderFactory {
156+
function stubProvider(trace: Trace): AgentProviderFactory {
115157
const rounds = new Map<Turn, number>();
116158
return function* (options) {
159+
trace.factoryOptions.push({ ...options });
117160
yield* Agent.around(
118161
{
119162
// deno-lint-ignore require-yield
120163
*agent([name]) {
164+
trace.agentSelections.push(name);
121165
return name ?? options.defaultAgent;
122166
},
123167
// deno-lint-ignore require-yield
124168
*session([name]) {
169+
trace.sessionSelections.push(name);
125170
return { sessionKey: `${name ?? "default"}`, cwd: "/" };
126171
},
127172
// deno-lint-ignore require-yield
128173
*prompt([content, promptOptions]) {
129174
const turn = classify(content);
130175
const round = (rounds.get(turn) ?? 0) + 1;
131176
rounds.set(turn, round);
132-
calls.push({ turn, session: sessionName(promptOptions), content });
177+
trace.calls.push({
178+
turn,
179+
session: sessionName(promptOptions),
180+
content,
181+
options: { ...(promptOptions ?? {}) },
182+
agent: String(promptOptions?.agent ?? options.defaultAgent),
183+
});
133184
return stream(reply(turn, round), promptOptions);
134185
},
135186
},
@@ -183,7 +234,6 @@ returns:
183234
instructions="INSTRUCTIONS"
184235
planner="planner"
185236
implementor="implementor"
186-
worktree="."
187237
as="planning"
188238
/>
189239
@@ -200,14 +250,15 @@ returns:
200250
}} />
201251
`;
202252

203-
function* runPlanning(): Operation<{ value: JsonObject; calls: Call[] }> {
204-
const calls: Call[] = [];
253+
function* runPlanning(): Operation<{ value: JsonObject; calls: Call[]; trace: Trace }> {
254+
const trace: Trace = { calls: [], factoryOptions: [], agentSelections: [], sessionSelections: [] };
255+
const calls = trace.calls;
205256
const dir = path.join(os.tmpdir(), `xmd-290-${randomUUID()}`);
206257
yield* ensureDir(dir);
207258
return yield* scoped(function* () {
208259
yield* ensure(() => rm(dir, { recursive: true, force: true }));
209260
const options: AgentProviderOptions = { defaultAgent: "stub", permissionMode: "deny-all" };
210-
yield* installAgentComponents({ rootProvider: { factory: stubProvider(calls), options } });
261+
yield* installAgentComponents({ rootProvider: { factory: stubProvider(trace), options } });
211262

212263
const root = path.join(dir, "exhaustion-root.md");
213264
yield* writeTextFile(root, ROOT);
@@ -228,7 +279,7 @@ function* runPlanning(): Operation<{ value: JsonObject; calls: Call[] }> {
228279
if (!isJsonObject(result.value)) {
229280
throw new Error(`the root returned ${JSON.stringify(result.value)}, not an object`);
230281
}
231-
return { value: result.value, calls };
282+
return { value: result.value, calls, trace };
232283
});
233284
}
234285

@@ -275,3 +326,158 @@ describe("#290 criterion 6 — an exhausted planning loop", () => {
275326
});
276327
});
277328
});
329+
330+
/**
331+
* The #302 boundary, proved over the same public root-provider seam.
332+
*
333+
* A root writes real instruction files, captures them with `InstructionFiles`,
334+
* and passes that captured text into `Discovery` and `Planning`. What an Agent
335+
* sees is then the whole question: the instruction paths and contents must
336+
* arrive, and the absolute directory they were read from must not — nor any
337+
* Workspace value, checkout path, directory registration or
338+
* `additionalDirectories` field.
339+
*/
340+
const BOUNDARY_ROOT = `---
341+
returns:
342+
handoff: { type: string }
343+
plan: { type: string }
344+
---
345+
346+
<InstructionFiles paths={["AGENTS.md", "nested/AGENTS.md"]} as="instructions" />
347+
348+
<Discovery
349+
instructions={instructions}
350+
planner="planner"
351+
request="Add a health endpoint"
352+
as="handoff"
353+
/>
354+
355+
<Planning
356+
handoff={handoff}
357+
handoffCheckpoint={{proceed: true, assessment: "HANDOFF-ASSESSED", response: "approved", rationale: "clear"}}
358+
instructions={instructions}
359+
planner="planner"
360+
implementor="implementor"
361+
as="planning"
362+
/>
363+
364+
<Return value={{ handoff: handoff, plan: planning.plan }} />
365+
`;
366+
367+
const ROOT_INSTRUCTION = "ROOT-INSTRUCTION prefer evidence over assertion.";
368+
const NESTED_INSTRUCTION = "NESTED-INSTRUCTION never edit a test to make it pass.";
369+
370+
function* runBoundary(
371+
inject?: (dir: string) => string,
372+
): Operation<{ trace: Trace; dir: string }> {
373+
const trace: Trace = { calls: [], factoryOptions: [], agentSelections: [], sessionSelections: [] };
374+
const dir = path.join(os.tmpdir(), `xmd-302-${randomUUID()}`);
375+
yield* ensureDir(path.join(dir, "nested"));
376+
return yield* scoped(function* () {
377+
yield* ensure(() => rm(dir, { recursive: true, force: true }));
378+
// The instructions are read relative to the contextual working directory,
379+
// exactly as a workflow run's `<Dir>` would establish it — so the document
380+
// names repository-relative paths and never an absolute one.
381+
yield* API.Env.around({ *cwd() { return dir; } }, { at: "min" });
382+
yield* useHostFiles();
383+
yield* writeTextFile(path.join(dir, "AGENTS.md"), `${ROOT_INSTRUCTION}\n`);
384+
yield* writeTextFile(path.join(dir, "nested", "AGENTS.md"), `${NESTED_INSTRUCTION}\n`);
385+
386+
const options: AgentProviderOptions = {
387+
// A mutation point: a host that leaked its work directory here would put
388+
// an absolute path in front of every Agent.
389+
defaultAgent: inject ? inject(dir) : "stub",
390+
permissionMode: "deny-all",
391+
};
392+
yield* installAgentComponents({ rootProvider: { factory: stubProvider(trace), options } });
393+
394+
const root = path.join(dir, "boundary-root.md");
395+
yield* writeTextFile(root, BOUNDARY_ROOT);
396+
const execution = yield* execute({
397+
path: root,
398+
stream: new InMemoryStream(),
399+
componentDirs: COMPONENT_DIRS,
400+
});
401+
const subscription = yield* execution.output;
402+
let next = yield* subscription.next();
403+
while (!next.done) {
404+
next = yield* subscription.next();
405+
}
406+
const result = yield* execution;
407+
if (!result.ok) {
408+
throw result.error;
409+
}
410+
return { trace, dir };
411+
});
412+
}
413+
414+
describe("#302 — an Agent reasons only over what a prompt renders", () => {
415+
it("carries instruction paths and contents to Discovery and both Planning prompts", function* () {
416+
const { trace } = yield* runBoundary();
417+
const seen = (turn: Turn) => trace.calls.filter((call) => call.turn === turn);
418+
419+
// Discovery, the plan prompt and the verdict prompt each receive both
420+
// repository-relative paths and both file contents, verbatim.
421+
for (const turn of ["discovery", "plan", "verdict"] as Turn[]) {
422+
const calls = seen(turn);
423+
expect(calls.length).toBeGreaterThan(0);
424+
for (const call of calls) {
425+
expect(call.content).toContain("AGENTS.md");
426+
expect(call.content).toContain("nested/AGENTS.md");
427+
expect(call.content).toContain(ROOT_INSTRUCTION);
428+
expect(call.content).toContain(NESTED_INSTRUCTION);
429+
}
430+
}
431+
});
432+
433+
it("lets no absolute directory, Workspace value or directory registration reach an Agent", function* () {
434+
const { trace, dir } = yield* runBoundary();
435+
const facing = agentFacingStrings(trace);
436+
437+
// The execution's own directory is where the instructions were read from.
438+
// It must not appear anywhere an Agent could observe.
439+
for (const value of facing) {
440+
expect(value).not.toContain(dir);
441+
expect(value).not.toContain(os.tmpdir());
442+
}
443+
444+
// Nor any directory-registration surface, by name or by field.
445+
const joined = facing.join("\n");
446+
for (const forbidden of [
447+
"additionalDirectories",
448+
"AddDir",
449+
"addDir",
450+
"workspaceRoot",
451+
"checkout",
452+
"materializ",
453+
]) {
454+
expect(joined).not.toContain(forbidden);
455+
}
456+
457+
// Prompt options carry only what agent selection needs.
458+
for (const call of trace.calls) {
459+
expect(Object.keys(call.options).sort()).toEqual(["agent", "session"]);
460+
}
461+
462+
// The repository-relative paths appear only inside the rendered
463+
// instruction material, never as a standalone Agent-facing input.
464+
const standalone = facing.filter((value) => value === "AGENTS.md" || value === "nested/AGENTS.md");
465+
expect(standalone).toHaveLength(0);
466+
});
467+
468+
it("trips the boundary guard when the execution's absolute directory is injected", function* () {
469+
const { trace, dir } = yield* runBoundary((workDir) => `stub-${workDir}`);
470+
471+
// Apply the guard the clean case relies on, and prove it fails here. If it
472+
// did not, the assertion above would be checking nothing.
473+
const leaked = agentFacingStrings(trace).filter((value) => value.includes(dir));
474+
expect(leaked.length).toBeGreaterThan(0);
475+
expect(() => {
476+
for (const value of agentFacingStrings(trace)) {
477+
if (value.includes(dir)) {
478+
throw new Error("absolute execution directory reached an Agent-facing input");
479+
}
480+
}
481+
}).toThrow("absolute execution directory reached an Agent-facing input");
482+
});
483+
});

0 commit comments

Comments
 (0)