|
| 1 | +// Adversarial / regression tests for the v0.8.0 release. |
| 2 | +// |
| 3 | +// Primary focus: the `reviewer` agent (new in #856). A v0.8.0 release review |
| 4 | +// found a P0 — the agent advertised "read-only / cannot modify files" but its |
| 5 | +// bash allowlist (`git log *`, `cat *`, `ls *`) was bypassable to arbitrary |
| 6 | +// file READ (exfil) and WRITE (shell redirects ride inside the matched |
| 7 | +// command). The fix denies bash entirely for the reviewer; these tests pin that |
| 8 | +// it stays denied and that the read-only intent holds. |
| 9 | +import { test, expect } from "bun:test" |
| 10 | +import { tmpdir } from "../fixture/fixture" |
| 11 | +import { Instance } from "../../src/project/instance" |
| 12 | +import { Agent } from "../../src/agent/agent" |
| 13 | +import { PermissionNext } from "../../src/permission/next" |
| 14 | + |
| 15 | +function bashAction(agent: Agent.Info, command: string) { |
| 16 | + return PermissionNext.evaluate("bash", command, agent.permission).action |
| 17 | +} |
| 18 | + |
| 19 | +test("reviewer agent: bash is denied (base)", async () => { |
| 20 | + await using tmp = await tmpdir() |
| 21 | + await Instance.provide({ |
| 22 | + directory: tmp.path, |
| 23 | + fn: async () => { |
| 24 | + const reviewer = await Agent.get("reviewer") |
| 25 | + expect(reviewer).toBeDefined() |
| 26 | + expect(PermissionNext.evaluate("bash", "*", reviewer!.permission).action).toBe("deny") |
| 27 | + }, |
| 28 | + }) |
| 29 | +}) |
| 30 | + |
| 31 | +test("reviewer agent: redirect-write and arbitrary-read bash commands are denied", async () => { |
| 32 | + await using tmp = await tmpdir() |
| 33 | + await Instance.provide({ |
| 34 | + directory: tmp.path, |
| 35 | + fn: async () => { |
| 36 | + const reviewer = (await Agent.get("reviewer"))! |
| 37 | + // Write via redirect riding inside a once-allowed `git log *` pattern. |
| 38 | + expect(bashAction(reviewer, "git log -p HEAD > ~/.ssh/authorized_keys")).toBe("deny") |
| 39 | + expect(bashAction(reviewer, "git diff HEAD >> ~/.bashrc")).toBe("deny") |
| 40 | + expect(bashAction(reviewer, "ls > /etc/cron.d/pwn")).toBe("deny") |
| 41 | + // Arbitrary file read (credential exfil) that `cat *` used to allow. |
| 42 | + expect(bashAction(reviewer, "cat ~/.altimate/altimate.json")).toBe("deny") |
| 43 | + expect(bashAction(reviewer, "cat .env")).toBe("deny") |
| 44 | + // Even the previously-allowed read-only git inspection is now denied. |
| 45 | + expect(bashAction(reviewer, "git log --oneline")).toBe("deny") |
| 46 | + }, |
| 47 | + }) |
| 48 | +}) |
| 49 | + |
| 50 | +test("reviewer agent: write/edit tools are denied, engine + read-only tools allowed", async () => { |
| 51 | + await using tmp = await tmpdir() |
| 52 | + await Instance.provide({ |
| 53 | + directory: tmp.path, |
| 54 | + fn: async () => { |
| 55 | + const reviewer = (await Agent.get("reviewer"))! |
| 56 | + const action = (perm: string) => PermissionNext.evaluate(perm, "*", reviewer.permission).action |
| 57 | + // Mutation denied — the verdict engine never writes. |
| 58 | + expect(action("edit")).toBe("deny") |
| 59 | + expect(action("write")).toBe("deny") |
| 60 | + expect(action("sql_execute_write")).toBe("deny") |
| 61 | + // The verdict engine + read-only analysis tools remain available. |
| 62 | + expect(action("dbt_pr_review")).toBe("allow") |
| 63 | + expect(action("read")).toBe("allow") |
| 64 | + expect(action("grep")).toBe("allow") |
| 65 | + expect(action("glob")).toBe("allow") |
| 66 | + }, |
| 67 | + }) |
| 68 | +}) |
| 69 | + |
| 70 | +test("reviewer agent is a selectable primary agent (not the default)", async () => { |
| 71 | + await using tmp = await tmpdir() |
| 72 | + await Instance.provide({ |
| 73 | + directory: tmp.path, |
| 74 | + fn: async () => { |
| 75 | + const names = (await Agent.list()).map((a) => a.name) |
| 76 | + expect(names).toContain("reviewer") |
| 77 | + // reviewer is selectable but must NOT hijack the default agent. |
| 78 | + expect(await Agent.defaultAgent()).not.toBe("reviewer") |
| 79 | + }, |
| 80 | + }) |
| 81 | +}) |
0 commit comments