Skip to content

Commit d17c5ae

Browse files
committed
test: exercise cloud handoffs through real SSH and TUI
1 parent 2db2a40 commit d17c5ae

7 files changed

Lines changed: 880 additions & 152 deletions

File tree

‎.github/workflows/cloud-acceptance.yml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,9 @@ jobs:
8989
sudo sysctl -w kernel.unprivileged_userns_clone=1
9090
fi
9191
92+
- name: Validate acceptance harness contracts
93+
run: python3 tests/test_cloud_acceptance_harness.py
94+
9295
- name: Check module declarations resolve
9396
run: python3 scripts/check_module_files.py
9497

‎docs/cloud-acceptance.md‎

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,11 @@ A workflow definition or successful build alone is not an acceptance result.
1818
The agent and TUI use `scripts/fake_openai_server.py`, a deterministic local
1919
OpenAI-compatible provider. It records requests and can hold/release turns.
2020
Provider context assertions test the handoff protocol, not model intelligence.
21+
The TUI submits real terminal keystrokes. It reads the client's debug `state`
22+
endpoint only to observe idle boundaries, never to seed or mutate client state.
23+
An old answer still present in terminal history cannot satisfy a new destination
24+
turn: the harness requires a fresh agent HTTP request with the cloud continuation.
25+
Connectivity probes and background memory extraction do not count as agent turns.
2126

2227
The TUI scenarios are:
2328

@@ -32,8 +37,10 @@ The TUI scenarios are:
3237
ownership intact, then a retry moves successfully.
3338
- `transfer_outage_retry`: an SSH failure after preparation preserves local
3439
ownership and epoch, then a retry moves successfully.
40+
- `destination_binary_retry`: missing and protocol-incompatible destination
41+
binaries fail before ownership changes, then a corrected destination works.
3542

36-
With the current six TUI scenarios, a full `--repeat 2` invocation records 18
43+
With the current seven TUI scenarios, a full `--repeat 2` invocation records 20
3744
stage results. Credential test cases are grouped into one stage per repetition.
3845
The runner continues after a failed stage and returns nonzero if any stage fails.
3946
Missing tools or failed namespace/bubblewrap probes are **failures, never skips**.
@@ -60,6 +67,16 @@ production tokens, account files or SSH keys to the fixtures or workflow.
6067

6168
## Manual use and focused diagnosis
6269

70+
Check the test machinery itself with:
71+
72+
```bash
73+
python3 tests/test_cloud_acceptance_harness.py
74+
```
75+
76+
These fast contracts check failure propagation, timeouts, report generation,
77+
provider gates, and fixture-only process cleanup. They are not a substitute for
78+
running the actual acceptance matrix below.
79+
6380
Use an already-built executable, with an absolute path recommended:
6481

6582
```bash

‎scripts/cloud_acceptance_tui.py‎

Lines changed: 472 additions & 0 deletions
Large diffs are not rendered by default.

‎scripts/fake_openai_server.py‎

Lines changed: 72 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,34 @@
11
#!/usr/bin/env python3
2-
"""Tiny OpenAI-compatible chat server for cloud-move e2e tests.
2+
"""Deterministic OpenAI-compatible provider for isolated acceptance tests.
33
4-
Each request is appended to $FAKE_LOG as JSON (the full message list the
5-
agent sent), and the reply echoes a short marker so tests can prove which
6-
machine produced which turn and what context the model saw.
4+
Only the model is substituted: tests still use Jcode's HTTP client, tools,
5+
transcripts, daemons, SSH transport and TUI. Optional file gates let a test
6+
hold a real streamed turn while submitting /cloud, without timing sleeps.
77
"""
8-
import json, os, sys, time
9-
from http.server import BaseHTTPRequestHandler, HTTPServer
8+
import json
9+
import os
10+
from pathlib import Path
11+
import re
12+
import sys
13+
import threading
14+
import time
15+
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
1016

11-
LOG = os.environ["FAKE_LOG"]
17+
LOG = os.environ.get("FAKE_LOG", "")
1218
LABEL = os.environ.get("FAKE_LABEL", "model")
19+
CONTROL = Path(os.environ["FAKE_CONTROL_DIR"]) if os.environ.get("FAKE_CONTROL_DIR") else None
20+
LOCK = threading.Lock()
21+
22+
23+
def text_of(message):
24+
content = message.get("content") or ""
25+
if isinstance(content, str):
26+
return content
27+
return "\n".join(part.get("text", "") for part in content if isinstance(part, dict))
1328

1429

1530
class H(BaseHTTPRequestHandler):
16-
def log_message(self, *a):
31+
def log_message(self, *args):
1732
pass
1833

1934
def do_GET(self):
@@ -27,16 +42,42 @@ def do_GET(self):
2742
def do_POST(self):
2843
req = json.loads(self.rfile.read(int(self.headers.get("content-length", 0))) or b"{}")
2944
msgs = req.get("messages", [])
30-
with open(LOG, "a") as f:
31-
f.write(json.dumps({"label": LABEL, "messages": msgs}) + "\n")
45+
with LOCK, open(LOG, "a") as log:
46+
log.write(json.dumps({"label": LABEL, "messages": msgs, "agent": bool(req.get("tools")),
47+
"time": time.time()}) + "\n")
3248
flat = json.dumps(msgs)
3349
seen_notice = "moved from" in flat and "to cloud host" in flat
3450
text = f"[{LABEL}] ack. saw_cloud_notice={seen_notice}"
51+
last_user = max((i for i, m in enumerate(msgs) if m.get("role") == "user"), default=-1)
52+
prompt = text_of(msgs[last_user]) if last_user >= 0 else ""
53+
has_tool_result = any(m.get("role") == "tool" for m in msgs[last_user + 1:])
54+
tool_calls = None
55+
if CONTROL and "[e2e:todo]" in prompt and not has_tool_result:
56+
# Go through the real tool executor, rather than mutating TUI state.
57+
tool_calls = [{"id": "call_acceptance_todo", "type": "function", "function": {
58+
"name": "todo", "arguments": json.dumps({
59+
"intent": "Create an open task for the handoff acceptance test",
60+
"todos": [{"id": "acceptance-open", "content": "Continue on cloud",
61+
"status": "in_progress", "priority": "high", "confidence": "plausible"}],
62+
})}}]
63+
elif CONTROL and (match := re.search(r"\[e2e:hold:([a-z0-9_-]+)\]", prompt)):
64+
token = match.group(1)
65+
CONTROL.mkdir(parents=True, exist_ok=True)
66+
(CONTROL / (token + ".held")).write_text("provider request received\n")
67+
deadline = time.monotonic() + 90
68+
while not (CONTROL / (token + ".release")).exists():
69+
if time.monotonic() >= deadline:
70+
self.send_error(504, "acceptance test did not release provider gate")
71+
return
72+
time.sleep(0.05)
3573
if not req.get("stream"):
74+
message = {"role": "assistant", "content": None if tool_calls else text}
75+
if tool_calls:
76+
message["tool_calls"] = tool_calls
3677
body = json.dumps({
3778
"id": "x", "object": "chat.completion", "model": "fake-model",
38-
"choices": [{"index": 0, "finish_reason": "stop",
39-
"message": {"role": "assistant", "content": text}}],
79+
"choices": [{"index": 0, "finish_reason": "tool_calls" if tool_calls else "stop",
80+
"message": message}],
4081
"usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
4182
}).encode()
4283
self.send_response(200)
@@ -48,17 +89,28 @@ def do_POST(self):
4889
self.send_response(200)
4990
self.send_header("content-type", "text/event-stream")
5091
self.end_headers()
51-
def ev(obj):
92+
93+
def event(obj):
5294
self.wfile.write(b"data: " + json.dumps(obj).encode() + b"\n\n")
5395
self.wfile.flush()
54-
ev({"id": "x", "object": "chat.completion.chunk", "model": "fake-model",
55-
"choices": [{"index": 0, "delta": {"role": "assistant", "content": text}}]})
56-
ev({"id": "x", "object": "chat.completion.chunk", "model": "fake-model",
57-
"choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}],
58-
"usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}})
96+
97+
delta = {"role": "assistant", "content": text}
98+
if tool_calls:
99+
delta = {"role": "assistant", "tool_calls": [dict(tool_calls[0], index=0)]}
100+
event({"id": "x", "object": "chat.completion.chunk", "model": "fake-model",
101+
"choices": [{"index": 0, "delta": delta}]})
102+
event({"id": "x", "object": "chat.completion.chunk", "model": "fake-model",
103+
"choices": [{"index": 0, "delta": {},
104+
"finish_reason": "tool_calls" if tool_calls else "stop"}],
105+
"usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}})
59106
self.wfile.write(b"data: [DONE]\n\n")
60107
self.wfile.flush()
61108

62109

63-
port = int(sys.argv[1])
64-
HTTPServer(("127.0.0.1", port), H).serve_forever()
110+
if __name__ == "__main__":
111+
if not LOG:
112+
raise SystemExit("FAKE_LOG must name an isolated request log")
113+
server = ThreadingHTTPServer(("127.0.0.1", int(sys.argv[1])), H)
114+
if os.environ.get("FAKE_PORT_FILE"):
115+
Path(os.environ["FAKE_PORT_FILE"]).write_text(str(server.server_port))
116+
server.serve_forever()

‎scripts/test_cloud_acceptance.py‎

Lines changed: 161 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,161 @@
1+
#!/usr/bin/env python3
2+
"""One gate for isolated cloud handoff acceptance. See docs/cloud-acceptance.md."""
3+
import argparse
4+
import json
5+
import os
6+
from pathlib import Path
7+
import shutil
8+
import signal
9+
import subprocess
10+
import sys
11+
import tempfile
12+
import time
13+
import traceback
14+
import xml.etree.ElementTree as ET
15+
16+
from cloud_acceptance_tui import CloudTUI, SCENARIOS
17+
18+
ROOT = Path(__file__).resolve().parent.parent
19+
20+
21+
def preflight(layers):
22+
required = {"bash", "git", "python3"}
23+
if "cli" in layers or "agent" in layers:
24+
required.add("bwrap")
25+
if "tui" in layers:
26+
required.update(("ssh", "sshd", "ssh-keygen", "tmux", "unshare", "mount"))
27+
missing = sorted(name for name in required if not shutil.which(name))
28+
if missing:
29+
raise RuntimeError("Missing test prerequisites: " + ", ".join(missing))
30+
if os.geteuid() == 0:
31+
raise RuntimeError("Run cloud acceptance as an ordinary user, not root")
32+
probes = []
33+
if "tui" in layers:
34+
probes.append(["unshare", "--user", "--map-root-user", "--mount", "true"])
35+
if "cli" in layers or "agent" in layers:
36+
probes.append(["bwrap", "--ro-bind", "/", "/", "--unshare-pid", "--proc", "/proc", "true"])
37+
for probe in probes:
38+
result = subprocess.run(probe, text=True, capture_output=True, timeout=10)
39+
if result.returncode:
40+
raise RuntimeError(f"Isolation unavailable ({' '.join(probe)}): {result.stderr.strip()}")
41+
42+
43+
def child(command, env, logfile, timeout):
44+
with logfile.open("w") as log:
45+
process = subprocess.Popen(command, cwd=ROOT, env=env, stdout=log,
46+
stderr=subprocess.STDOUT, start_new_session=True)
47+
try:
48+
code = process.wait(timeout=timeout)
49+
except subprocess.TimeoutExpired:
50+
os.killpg(process.pid, signal.SIGTERM)
51+
try:
52+
process.wait(timeout=10)
53+
except subprocess.TimeoutExpired:
54+
os.killpg(process.pid, signal.SIGKILL)
55+
process.wait()
56+
raise AssertionError(f"Stage timed out after {timeout}s. See {logfile}")
57+
if code:
58+
raise AssertionError(f"Stage exited {code}. See {logfile}")
59+
60+
61+
def write_report(path, results, binary, version):
62+
report = {"binary": binary, "version": version, "results": results,
63+
"limitations": ["No hosted VM provisioning/suspend/resume or production OAuth service is exercised.",
64+
"The provider is deterministic; SSH, TUI, daemons and transfer code are real."]}
65+
(path / "report.json").write_text(json.dumps(report, indent=2) + "\n")
66+
suite = ET.Element("testsuite", name="cloud-acceptance", tests=str(len(results)),
67+
failures=str(sum(r["status"] != "passed" for r in results)))
68+
for result in results:
69+
case = ET.SubElement(suite, "testcase", name=result["name"], time=str(result["seconds"]))
70+
if result["status"] != "passed":
71+
ET.SubElement(case, "failure", message=result["error"]).text = result.get("traceback", "")
72+
ET.SubElement(case, "system-out").text = result["artifacts"]
73+
ET.ElementTree(suite).write(path / "junit.xml", encoding="utf-8", xml_declaration=True)
74+
75+
76+
def main():
77+
parser = argparse.ArgumentParser(description=__doc__)
78+
parser.add_argument("--binary", default=os.environ.get("JCODE_BIN"), required=not os.environ.get("JCODE_BIN"))
79+
parser.add_argument("--layers", nargs="+", choices=("cli", "agent", "credentials", "tui"),
80+
default=["cli", "agent", "credentials", "tui"])
81+
parser.add_argument("--scenario", choices=tuple(SCENARIOS), help="Run one TUI scenario (with --layers tui)")
82+
parser.add_argument("--repeat", type=int, default=1, help="Repeat each selected stage to expose state leakage")
83+
parser.add_argument("--artifacts", type=Path, help="New directory for retained logs, homes and reports")
84+
parser.add_argument("--timeout", type=int, default=240, help="Timeout per CLI/agent/credential stage")
85+
args = parser.parse_args()
86+
if args.repeat < 1 or args.timeout < 1:
87+
parser.error("--repeat and --timeout must be positive")
88+
binary = str(Path(args.binary).resolve(strict=True))
89+
if args.artifacts:
90+
artifacts = args.artifacts.resolve()
91+
artifacts.mkdir(parents=True, exist_ok=False)
92+
else:
93+
parent = os.environ.get("JCODE_SCRATCH_DIR", tempfile.gettempdir())
94+
artifacts = Path(tempfile.mkdtemp(prefix="cloud-acceptance-", dir=parent))
95+
os.chmod(artifacts, 0o700)
96+
results = []
97+
version = subprocess.run([binary, "--no-update", "--version"], capture_output=True,
98+
text=True, timeout=10).stdout.strip()
99+
print(f"Testing {version}: {binary}\nArtifacts: {artifacts}", flush=True)
100+
try:
101+
preflight(args.layers)
102+
except Exception as exc:
103+
results.append({"name": "preflight", "status": "failed", "seconds": 0,
104+
"error": str(exc), "artifacts": str(artifacts)})
105+
write_report(artifacts, results, binary, version)
106+
print(f"FAIL preflight: {exc}", file=sys.stderr)
107+
return 1
108+
for iteration in range(1, args.repeat + 1):
109+
stages = [(layer, None) for layer in args.layers if layer != "tui"]
110+
if "tui" in args.layers:
111+
stages += [("tui", name) for name in ([args.scenario] if args.scenario else SCENARIOS)]
112+
for layer, scenario in stages:
113+
name = f"{layer}.{scenario or 'suite'}.{iteration}"
114+
directory = artifacts / name
115+
started = time.monotonic()
116+
record = {"name": name, "artifacts": str(directory), "status": "passed"}
117+
fixture = None
118+
print(f"RUN {name}", flush=True)
119+
try:
120+
if layer == "tui":
121+
fixture = CloudTUI(binary, directory)
122+
fixture.start()
123+
SCENARIOS[scenario](fixture)
124+
else:
125+
directory.mkdir()
126+
# Do not pass caller session/account/transport state to a test.
127+
env = {"PATH": os.environ.get("PATH", "/usr/bin:/bin"),
128+
"HOME": str(directory), "LANG": "C.UTF-8", "JCODE_BIN": binary,
129+
"JCODE_SCRATCH_DIR": str(directory), "TMPDIR": str(directory),
130+
"JCODE_NO_AUTO_UPDATE": "1", "JCODE_NO_TELEMETRY": "1"}
131+
script = {"cli": "scripts/test_cloud_move_e2e.sh",
132+
"agent": "scripts/test_cloud_move_agent_e2e.sh",
133+
"credentials": "tests/test_cloud_credentials.py"}[layer]
134+
command = ([sys.executable, script, binary] if layer == "credentials"
135+
else ["bash", script])
136+
child(command, env, directory / "output.log", args.timeout)
137+
except Exception as exc:
138+
record.update(status="failed", error=str(exc), traceback=traceback.format_exc())
139+
print(f"FAIL {name}: {exc}", flush=True)
140+
finally:
141+
if fixture:
142+
try:
143+
fixture.close()
144+
except Exception as exc:
145+
cleanup_error = f"Cleanup failed: {exc}"
146+
record.update(status="failed",
147+
error=(record["error"] + "\n" + cleanup_error
148+
if "error" in record else cleanup_error),
149+
traceback=record.get("traceback", "") + traceback.format_exc())
150+
record["seconds"] = round(time.monotonic() - started, 3)
151+
results.append(record)
152+
write_report(artifacts, results, binary, version)
153+
if record["status"] == "passed":
154+
print(f"PASS {name} ({record['seconds']}s)", flush=True)
155+
failed = sum(result["status"] != "passed" for result in results)
156+
print(f"\n{len(results) - failed} passed, {failed} failed. Report: {artifacts / 'report.json'}", flush=True)
157+
return int(failed != 0)
158+
159+
160+
if __name__ == "__main__":
161+
sys.exit(main())

0 commit comments

Comments
 (0)