|
| 1 | +#!/usr/bin/env bash |
| 2 | +# |
| 3 | +# Live end-to-end check: drive the real OpenCode CLI against allowlister wired as |
| 4 | +# a `tool.execute.before` plugin shim, and assert that a denied command is blocked |
| 5 | +# and an allowed command runs. |
| 6 | +# |
| 7 | +# This is deliberately NOT part of `just full-check` or CI: it needs the |
| 8 | +# `opencode` binary, network access, an LLM provider key, and a (cheap) model |
| 9 | +# call, so it is neither hermetic nor deterministic the way the `tests/` suite is. |
| 10 | +# Run it by hand (or via `just test-opencode`) to verify the integration against a |
| 11 | +# real harness after changing the adapter or the plugin shim. |
| 12 | +# |
| 13 | +# What it proves, using the command's side effect (a sentinel file): |
| 14 | +# * deny -> the command never executes (sentinel is absent) EVEN under |
| 15 | +# `--dangerously-skip-permissions`: that flag only relaxes OpenCode's |
| 16 | +# own permission system; our plugin's `throw` fires regardless, so it |
| 17 | +# is the sole gate. This is the core security claim. |
| 18 | +# * allow -> the command executes (sentinel is written with its marker): an |
| 19 | +# allow verdict emits nothing, so the plugin does not throw. |
| 20 | +# The hook's reason string is checked best-effort: OpenCode's event schema is not |
| 21 | +# pinned here, so a missing reason is a note, not a failure. The side effects are |
| 22 | +# the hard assertions. |
| 23 | +# |
| 24 | +# `defer` is not asserted: it lets the call proceed, which has no deterministic |
| 25 | +# headless outcome. That path is covered hermetically in tests/e2e. |
| 26 | +# |
| 27 | +# Environment overrides: |
| 28 | +# ALLOWLISTER_E2E_MODEL model passed to `opencode run -m` (default: anthropic/claude-haiku-4-5) |
| 29 | +# ALLOWLISTER_E2E_KEEP set to 1 to keep the temp sandbox for inspection |
| 30 | +# OPENCODE_BIN opencode binary name/path (default: opencode) |
| 31 | +# ANTHROPIC_API_KEY / provider key OpenCode uses to run the model (match it |
| 32 | +# OPENAI_API_KEY to ALLOWLISTER_E2E_MODEL's provider) |
| 33 | + |
| 34 | +set -euo pipefail |
| 35 | + |
| 36 | +agent_bin="${OPENCODE_BIN:-opencode}" |
| 37 | +model="${ALLOWLISTER_E2E_MODEL:-anthropic/claude-haiku-4-5}" |
| 38 | +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" |
| 39 | +bin="$repo_root/target/release/allowlister" |
| 40 | + |
| 41 | +note() { printf '%s\n' "$*"; } |
| 42 | +fail() { printf 'FAIL: %s\n' "$*" >&2; exit 1; } |
| 43 | + |
| 44 | +# A missing `opencode` is a skip, not a failure: this script is opt-in and the |
| 45 | +# rest of the project must build and test on machines without the harness. |
| 46 | +if ! command -v "$agent_bin" >/dev/null 2>&1; then |
| 47 | + note "SKIP: \`$agent_bin\` not found on PATH (install OpenCode — \`npm i -g opencode-ai\` — to run this check)." |
| 48 | + exit 0 |
| 49 | +fi |
| 50 | + |
| 51 | +note "» building release binary" |
| 52 | +( cd "$repo_root" && cargo build --release --locked --quiet ) |
| 53 | +[ -x "$bin" ] || fail "release binary not found at $bin" |
| 54 | + |
| 55 | +# allowlister must be on PATH so the plugin shim's `allowlister hook opencode` |
| 56 | +# resolves when OpenCode spawns it. The plugin inherits this process's PATH. |
| 57 | +bindir="$repo_root/target/release" |
| 58 | +export PATH="$bindir:$PATH" |
| 59 | + |
| 60 | +sandbox="$(mktemp -d)" |
| 61 | +cleanup() { [ "${ALLOWLISTER_E2E_KEEP:-0}" = "1" ] || rm -rf "$sandbox"; } |
| 62 | +trap cleanup EXIT |
| 63 | + |
| 64 | +proj="$sandbox/project" |
| 65 | +mkdir -p "$proj/.git" |
| 66 | + |
| 67 | +# Isolate OpenCode's user state under the sandbox so no ambient ~/.config/opencode |
| 68 | +# plugins or data leak in. Auth comes from the provider env key, so no auth.json |
| 69 | +# is needed. The project plugin under <proj>/.opencode/plugin is discovered from |
| 70 | +# the cwd regardless of HOME. |
| 71 | +export HOME="$sandbox/home" |
| 72 | +mkdir -p "$HOME" |
| 73 | + |
| 74 | +# Deterministic rules: deny `touch`, allow `mkdir`. The allow case is a |
| 75 | +# redirect-free command so a headless model reproduces it verbatim — some models |
| 76 | +# silently drop a `> file` redirection, which would make the allow case flaky for |
| 77 | +# reasons unrelated to the gate. (Redirection policy is covered hermetically in |
| 78 | +# the unit and tests/e2e suites.) |
| 79 | +rules="$sandbox/rules.json" |
| 80 | +cat > "$rules" <<JSON |
| 81 | +{ |
| 82 | + "rules": [ |
| 83 | + { "name": "deny touch", "match": "touch *", "action": "deny" }, |
| 84 | + { "name": "allow mkdir", "match": "mkdir *", "action": "allow" } |
| 85 | + ] |
| 86 | +} |
| 87 | +JSON |
| 88 | + |
| 89 | +# Set the project up exactly the way a user would: `init` writes the project |
| 90 | +# `.allowlister.json` (here from our deterministic rules file) AND writes the |
| 91 | +# OpenCode plugin shim into .opencode/plugin/. Exercising init here means the live |
| 92 | +# check also covers the plugin-write path end to end. |
| 93 | +note "» wiring the project with \`allowlister init --harness opencode\`" |
| 94 | +( cd "$proj" && "$bin" init --local --profile "$rules" --harness opencode --hooks --force ) >/dev/null \ |
| 95 | + || fail "allowlister init failed to set the project up" |
| 96 | +[ -f "$proj/.allowlister.json" ] || fail "init did not write the project config" |
| 97 | +grep -q 'allowlister hook opencode' "$proj/.opencode/plugin/allowlister.js" \ |
| 98 | + || fail "init did not write the OpenCode plugin shim" |
| 99 | + |
| 100 | +# Run one headless turn steered toward a single exact command. |
| 101 | +# * `opencode run` is the non-interactive entry point. |
| 102 | +# * --dangerously-skip-permissions relaxes OpenCode's OWN permission system (it |
| 103 | +# otherwise auto-rejects every request headless). Our plugin's throw fires |
| 104 | +# regardless, so this makes the deny case a true test of the plugin's authority. |
| 105 | +# * -m sets the model; --format json keeps output machine-readable. |
| 106 | +# * stdin from /dev/null avoids any interactive "waiting for stdin" delay. |
| 107 | +run_agent() { |
| 108 | + local prompt="$1" stream="$2" |
| 109 | + ( cd "$proj" && timeout 180 "$agent_bin" run \ |
| 110 | + --dangerously-skip-permissions \ |
| 111 | + --format json \ |
| 112 | + -m "$model" \ |
| 113 | + "$prompt" \ |
| 114 | + </dev/null ) >"$stream" 2>"$stream.err" || { |
| 115 | + note " ($agent_bin exited non-zero; stderr tail:)"; tail -3 "$stream.err" >&2 || true |
| 116 | + } |
| 117 | +} |
| 118 | + |
| 119 | +# True if allowlister's own reason text reached the OpenCode transcript. OpenCode |
| 120 | +# may render the block differently, so this is a bonus signal, not a gate. |
| 121 | +reason_surfaced_in() { grep -aq 'allowlister' "$1" "$1.err" 2>/dev/null; } |
| 122 | + |
| 123 | +# Diagnostic: print a transcript so a CI run reveals exactly how OpenCode handled |
| 124 | +# the turn — a plugin throw on deny, or what the model actually ran on allow. |
| 125 | +dump_transcript() { |
| 126 | + local stream="$1" label="$2" |
| 127 | + note " ── $label transcript diagnostic ─────────────────────────" |
| 128 | + note " stdout (<=200 lines):" |
| 129 | + head -200 "$stream" | sed 's/^/ /' |
| 130 | + note " stderr tail:" |
| 131 | + tail -12 "$stream.err" 2>/dev/null | sed 's/^/ /' |
| 132 | + note " ────────────────────────────────────────────────────────" |
| 133 | +} |
| 134 | + |
| 135 | +note "» case 1/2: deny — \`touch\` must be blocked even under --dangerously-skip-permissions" |
| 136 | +deny_sentinel="$sandbox/sentinel-deny.txt" |
| 137 | +rm -f "$deny_sentinel" |
| 138 | +run_agent "Use the shell to run exactly this one command, then stop: touch $deny_sentinel — if it is blocked, stop immediately and do not try any alternative." \ |
| 139 | + "$sandbox/deny.stream" |
| 140 | +[ -e "$deny_sentinel" ] && { |
| 141 | + dump_transcript "$sandbox/deny.stream" deny |
| 142 | + fail "denied command executed: $deny_sentinel was created (the plugin throw did not hold)" |
| 143 | +} |
| 144 | +note " ok: command blocked — the sentinel was never created" |
| 145 | +if reason_surfaced_in "$sandbox/deny.stream"; then |
| 146 | + note " bonus: allowlister's reason text reached the OpenCode transcript" |
| 147 | +else |
| 148 | + note " note: allowlister's reason was not surfaced in the transcript (OpenCode rendered its own error)" |
| 149 | +fi |
| 150 | + |
| 151 | +note "» case 2/2: allow — \`mkdir\` must run" |
| 152 | +allow_sentinel="$proj/sentinel-allow.d" |
| 153 | +rm -rf "$allow_sentinel" |
| 154 | +run_agent "Use the shell to run exactly this one command, then stop: mkdir $allow_sentinel" \ |
| 155 | + "$sandbox/allow.stream" |
| 156 | +[ -d "$allow_sentinel" ] || { |
| 157 | + dump_transcript "$sandbox/allow.stream" allow |
| 158 | + fail "allowed command did not execute: $allow_sentinel was not created" |
| 159 | +} |
| 160 | +note " ok: command executed (allow did not trip the plugin)" |
| 161 | + |
| 162 | +note "✓ opencode live e2e passed (deny blocked under --dangerously-skip-permissions, allow ran)" |
0 commit comments