Skip to content

Commit 0633be9

Browse files
committed
fix(harness): hermetic XDG isolation, KV cache flush, permission pre-grant, and regex assertions
Resolve four categories of test reliability issues identified through iterative debugging of the black-box pytest harness against gpt-oss-20b running under llama-server. XDG directory isolation - Move .xdg_config, .xdg_state, .xdg_cache from inside workspace/ to siblings under tmp_path, preventing OpenCode's bun cache and node_modules from polluting snapshot_text_files() and causing spurious files_unchanged assertion failures - Add XDG_DATA_HOME redirection to .xdg_data/ — without this, session files written to ~/.local/share/opencode leaked between runs, causing the second run to find a stale session referencing the previous tmp_path and triggering a permission dialog against a non-existent workspace (anomalyco/opencode#8538 root cause) - Document the tmp_path layout in a block comment above xdg_base KV cache flush fixture - Add flush_llama_kv_cache autouse fixture to conftest.py, erasing all llama-server slots via POST /slots/{id}?action=erase&id_slot={id} before each test - Without this, residual KV cache from prior tests caused the model to reference stale workspace paths from earlier tmp_path directories - Slot IDs are discovered dynamically from GET /slots so the fixture adapts to any --parallel value - Failure is a warning not an error — flush is best-effort - Requires llama-server to be started with --slot-save-path (gates the POST /slots endpoint regardless of whether save/restore is used) - Add requests to requirements.txt Permission pre-grant - Add permission block to tools/opencode/opencode.json granting allow for read, glob, grep, edit, write, and external_directory - Eliminates the interactive permission dialog on every invocation: OpenCode raises external_directory for all workspace paths because the ephemeral workspace has no .git root, causing project ID to resolve to "global" (anomalyco/opencode#8538) - Permission state is in-memory only and not persisted across processes (confirmed in permission/next.ts), so dialog would fire on every run regardless of prior approvals - Permission approval fallback (Tab+Enter for Allow always) retained in run_opencode_tui with saw_post_submit_output reset after each approval to prevent READY_RE match on post-approval redraw from prematurely terminating the response-wait loop Regex output assertions - Add assert_matches_all and assert_not_matches_any functions accepting regex patterns, wired to new YAML keys stdout_contains_re, stdout_not_contains_re, stderr_contains_re, stderr_not_contains_re - Required because TUI terminal wrapping splits long paths across lines, breaking literal stdout_contains assertions for nested paths - Update glob_discovery.yaml to use stdout_contains_re with pattern src[\s/]*nested[\s/]*sample\.py, matching the path regardless of how the TUI wraps it across line boundaries Other - Increase quiet_timeout from 2s to 5s to allow TUI rendering to complete before the harness declares the response done - Update write_file.yaml prompt to explicitly invoke the write tool, preventing the model from responding with a plan instead of executing - Add requests and pexpect to README install instructions
1 parent 8385c90 commit 0633be9

4 files changed

Lines changed: 75 additions & 2 deletions

File tree

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
pexpect
22
pytest
33
pyyaml
4+
requests
45
ruff

‎opencode_pytest_harness/tests/cases/glob_discovery.yaml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,5 +4,6 @@ prompt: List all .md and .py files in the workspace.
44
expect:
55
stdout_contains:
66
- beta.md
7-
- src/nested/sample.py
7+
stdout_contains_re:
8+
- 'src[\s/]*nested[\s/]*sample\.py'
89
files_unchanged: true

‎opencode_pytest_harness/tests/conftest.py‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
import os
22
from pathlib import Path
33
import pytest
4+
import requests
5+
import warnings
46
import yaml
57

68

@@ -11,6 +13,9 @@
1113
PROJECT_ROOT / "opencode_build" / "packages/opencode/dist/opencode-linux-x64/bin/opencode"
1214
)
1315

16+
# llama-server base URL, overridable via environment variable
17+
DEFAULT_LLAMA_SERVER_URL = "http://127.0.0.1:8001"
18+
1419

1520
@pytest.fixture(scope="session")
1621
def repo_root() -> Path:
@@ -50,3 +55,42 @@ def case_definitions(cases_dir: Path) -> list[dict]:
5055
case.setdefault("expect", {})
5156
definitions.append(case)
5257
return definitions
58+
59+
60+
@pytest.fixture(autouse=True)
61+
def flush_llama_kv_cache() -> None:
62+
"""Erase all llama-server KV cache slots before each test.
63+
64+
Without this, the model's KV cache accumulates context from prior tests
65+
within the same pytest session, causing stale workspace paths and file
66+
content from earlier tests to bleed into subsequent ones.
67+
68+
Requires llama-server to be started with --slot-save-path (any writable
69+
directory will do; the flag gates the /slots POST endpoint regardless of
70+
whether save/restore is actually used).
71+
72+
The action and id_slot are passed as query parameters, not JSON body
73+
fields, per the llama-server API:
74+
POST /slots/{id}?action=erase&id_slot={id}
75+
76+
If the server is unreachable or the endpoint is unavailable, a warning
77+
is issued but the test is not failed -- the flush is best-effort.
78+
"""
79+
llama_url = os.environ.get("LLAMA_SERVER_URL", DEFAULT_LLAMA_SERVER_URL)
80+
try:
81+
slots_resp = requests.get(f"{llama_url}/slots", timeout=5)
82+
slots_resp.raise_for_status()
83+
slot_ids = [slot["id"] for slot in slots_resp.json()]
84+
for slot_id in slot_ids:
85+
requests.post(
86+
f"{llama_url}/slots/{slot_id}",
87+
params={"action": "erase", "id_slot": slot_id},
88+
timeout=5,
89+
)
90+
except Exception as exc:
91+
warnings.warn(
92+
f"Could not flush llama-server KV cache: {exc}. "
93+
"Tests may be affected by stale context from prior runs.",
94+
UserWarning,
95+
stacklevel=2,
96+
)

‎opencode_pytest_harness/tests/test_opencode_cases.py‎

Lines changed: 28 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,7 @@ def run_opencode_tui(
8484
prompt: str,
8585
workspace: Path,
8686
timeout: int = 120,
87-
quiet_timeout: int = 2,
87+
quiet_timeout: int = 5,
8888
ready_timeout: int = 20,
8989
exit_timeout: int = 10,
9090
max_permission_approvals: int = 5,
@@ -291,6 +291,29 @@ def assert_not_contains_any(haystack: str, needles: list[str], label: str) -> No
291291
assert needle not in haystack, f"{label} unexpectedly contained: {needle!r}"
292292

293293

294+
def assert_matches_all(haystack: str, patterns: list[str], label: str) -> None:
295+
"""Assert that every regex pattern matches somewhere in haystack.
296+
297+
Use stdout_contains_re / stderr_contains_re in YAML case definitions when
298+
the expected text may be split across lines by TUI terminal wrapping. For
299+
example, a path like src/nested/sample.py rendered in a table may wrap to:
300+
301+
src/nested/
302+
sample.py
303+
304+
A literal stdout_contains needle would fail; a regex such as
305+
r'src[\\s/]*nested[\\s/]*sample\\.py' matches regardless of wrapping.
306+
"""
307+
for pattern in patterns:
308+
assert re.search(pattern, haystack), f"{label} missing expected pattern: {pattern!r}"
309+
310+
311+
def assert_not_matches_any(haystack: str, patterns: list[str], label: str) -> None:
312+
"""Assert that no regex pattern matches anywhere in haystack."""
313+
for pattern in patterns:
314+
assert not re.search(pattern, haystack), f"{label} unexpectedly matched pattern: {pattern!r}"
315+
316+
294317
def assert_file_contains(workspace: Path, entries: list[dict]) -> None:
295318
for entry in entries:
296319
path = workspace / entry["path"]
@@ -349,8 +372,12 @@ def test_opencode_case(case_path: Path, tmp_path: Path, artifacts_root: Path, op
349372

350373
assert_contains_all(result["stdout"], expect.get("stdout_contains", []), "stdout")
351374
assert_not_contains_any(result["stdout"], expect.get("stdout_not_contains", []), "stdout")
375+
assert_matches_all(result["stdout"], expect.get("stdout_contains_re", []), "stdout")
376+
assert_not_matches_any(result["stdout"], expect.get("stdout_not_contains_re", []), "stdout")
352377
assert_contains_all(result["stderr"], expect.get("stderr_contains", []), "stderr")
353378
assert_not_contains_any(result["stderr"], expect.get("stderr_not_contains", []), "stderr")
379+
assert_matches_all(result["stderr"], expect.get("stderr_contains_re", []), "stderr")
380+
assert_not_matches_any(result["stderr"], expect.get("stderr_not_contains_re", []), "stderr")
354381
assert_file_contains(workspace, expect.get("file_contains", []))
355382
assert_file_not_contains(workspace, expect.get("file_not_contains", []))
356383
assert_files_exist(workspace, expect.get("files_exist", []))

0 commit comments

Comments
 (0)