Skip to content

Instantly share code, notes, and snippets.

Show Gist options
  • Select an option

  • Save N0zoM1z0/5861579c2e83fed87126293bf76bbbcf to your computer and use it in GitHub Desktop.

Select an option

Save N0zoM1z0/5861579c2e83fed87126293bf76bbbcf to your computer and use it in GitHub Desktop.
modelock self-contained reproducer: opencode-post-tool-transient-disconnect
#!/usr/bin/env python3
"""Standalone vendor reproducer for CLI provider-stream bugs.
Usage:
python3 reproducer.py case.json
The script intentionally imports only Python standard-library modules. It
starts a local mock provider, runs the target CLI in Docker, records provider
requests, and evaluates simple case assertions.
"""
from __future__ import annotations
import argparse
import json
import os
from pathlib import Path
import shlex
import signal
import subprocess
import sys
import tempfile
import threading
import time
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from typing import Any
PROMPT = "Briefly inspect this directory and use the requested tool if needed."
DOCKER_PATH = "/home/runner/.npm-global/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
DEFAULT_DOCKER_CPUS = "2"
DEFAULT_DOCKER_MEMORY = "4g"
EMBEDDED_CASE: dict[str, Any] | None = json.loads(r'''
{
"checks": [
{
"kind": "provider_requests_at_least",
"min_count": 3
},
{
"kind": "closed_mid_stream_at_least",
"min_count": 1
},
{
"kind": "nonzero_or_timeout"
}
],
"close_after_events": 4,
"image": "vendor-repro-opencode:1.17.4",
"steps": [
{
"provider.stream.tool_call_start": {
"id": "disconnect_probe",
"name": "Bash"
}
},
{
"provider.stream.tool_args_delta": {
"delta": "{\"command\":\"printf MODELOCK_DISCONNECT_MID_TOOL\\\\n\"}",
"id": "disconnect_probe"
}
},
{
"provider.stream.tool_call_stop": {
"id": "disconnect_probe"
}
},
{
"provider.stream.message_stop": {}
}
],
"target": "opencode",
"timeout_s": 75,
"version": "1.17.4",
"workspace_files": {
"src/app.js": "console.log('modelock');\n"
}
}
''')
def dockerfile_for_target(target: str, version: str) -> str:
common = """FROM node:24-bookworm
ARG RUNNER_UID=1000
ARG RUNNER_GID=1000
ENV DEBIAN_FRONTEND=noninteractive
ENV NPM_CONFIG_PREFIX=/home/runner/.npm-global
ENV PATH=/home/runner/.npm-global/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
RUN apt-get update && apt-get install -y --no-install-recommends \\
python3 git curl ca-certificates procps lsof ripgrep passwd \\
&& rm -rf /var/lib/apt/lists/*
RUN set -eux; \\
if ! getent group "${RUNNER_GID}" >/dev/null; then groupadd -g "${RUNNER_GID}" runner; fi; \\
if ! getent passwd "${RUNNER_UID}" >/dev/null; then \\
useradd -m -u "${RUNNER_UID}" -g "${RUNNER_GID}" -d /home/runner -s /bin/bash runner; \\
else \\
existing_user="$(getent passwd "${RUNNER_UID}" | cut -d: -f1)"; \\
usermod -d /home/runner "${existing_user}" || true; \\
fi; \\
mkdir -p /home/runner/.npm-global; \\
chown -R "${RUNNER_UID}:${RUNNER_GID}" /home/runner; \\
chmod -R a+rx /home/runner
"""
if target == "claude":
install = f"RUN npm install -g @anthropic-ai/claude-code@{version} && chmod -R a+rx /home/runner\n"
elif target == "codex":
install = f"RUN npm install -g @openai/codex@{version} && chmod -R a+rx /home/runner\n"
elif target == "qwen":
install = f"RUN npm install -g @qwen-code/qwen-code@{version} && chmod -R a+rx /home/runner\n"
elif target == "crush":
install = f"RUN npm install -g @charmland/crush@{version} && chmod -R a+rx /home/runner\n"
elif target == "goose":
install = (
"RUN curl -fsSL --retry 3 --connect-timeout 20 --max-time 180 "
"https://github.com/aaif-goose/goose/releases/download/stable/download_cli.sh "
f"| GOOSE_VERSION=v{version} CONFIGURE=false GOOSE_BIN_DIR=/usr/local/bin bash\n"
)
elif target == "cline":
install = (
f"RUN npm install -g cline@{version} "
f"&& cline_root=\"$(npm root -g)/@cline/cli-linux-x64\" "
"&& if [ ! -x \"$cline_root/bin/cline\" ]; then "
"tmp=\"$(mktemp -d)\"; "
f"curl -fsSL --retry 3 https://registry.npmjs.org/@cline/cli-linux-x64/-/cli-linux-x64-{version}.tgz -o \"$tmp/cline.tgz\"; "
"tar -xzf \"$tmp/cline.tgz\" -C \"$tmp\"; "
"mkdir -p \"$cline_root/bin\"; "
"cp \"$tmp/package/bin/cline\" \"$cline_root/bin/cline\"; "
"chmod +x \"$cline_root/bin/cline\"; "
"fi "
"&& chmod -R a+rx /home/runner\n"
)
elif target == "opencode":
install = f"RUN npm install -g opencode-ai@{version} && chmod -R a+rx /home/runner\n"
elif target == "continue":
install = f"RUN npm install -g @continuedev/cli@{version} && chmod -R a+rx /home/runner\n"
else:
raise ValueError(f"unsupported target for Dockerfile: {target}")
return common + install + 'WORKDIR /workspace\nENTRYPOINT ["/usr/bin/env"]\nCMD ["bash"]\n'
class ProviderState:
def __init__(self, case: dict[str, Any]) -> None:
self.case = case
self.requests: list[dict[str, Any]] = []
self.lock = threading.Lock()
def append_request(self, entry: dict[str, Any]) -> None:
with self.lock:
self.requests.append(entry)
def request_count(self) -> int:
with self.lock:
return len(self.requests)
def snapshot_requests(self) -> list[dict[str, Any]]:
with self.lock:
return list(self.requests)
class ProviderServer(ThreadingHTTPServer):
def __init__(self, address: tuple[str, int], state: ProviderState) -> None:
self.state = state
super().__init__(address, ProviderHandler)
class ProviderHandler(BaseHTTPRequestHandler):
server: ProviderServer
def log_message(self, fmt: str, *args: Any) -> None:
return None
def do_GET(self) -> None:
if self.path in {"/health", "/__repro/health"}:
self._json(200, {"ok": True})
return
if self.path in {"/v1/models", "/v1/anthropic/models"}:
self._json(
200,
{
"object": "list",
"data": [
{"id": "gpt-4", "object": "model"},
{"id": "claude-3-5-sonnet-20241022", "type": "model"},
{"id": "claude-sonnet-4-5", "type": "model"},
],
},
)
return
self._json(404, {"error": "not found"})
def do_POST(self) -> None:
started = time.time()
body = self._read_json()
requests_before = self.server.state.request_count()
response = select_response(self.server.state.case, body, requests_before)
entry = {
"ts": started,
"path": self.path,
"body": body,
"response_events": [],
"status": response.get("status", 200),
"closed_mid_stream": False,
}
self.server.state.append_request(entry)
if response.get("kind") == "error":
self._json(int(response.get("status", 500)), {"error": response.get("error", {})})
return
wire = wire_for_path(self.path)
events = encode_stream(self.server.state.case, response.get("steps", []), wire, str(body.get("model", "")))
close_after = response.get("close_after_events")
self._sse(entry, events, int(close_after) if close_after is not None else None)
def _read_json(self) -> dict[str, Any]:
length = int(self.headers.get("content-length") or 0)
if length <= 0:
return {}
raw = self.rfile.read(length)
try:
return json.loads(raw.decode("utf-8"))
except json.JSONDecodeError:
return {"_raw": raw.decode("utf-8", errors="replace")}
def _json(self, status: int, payload: dict[str, Any]) -> None:
raw = json.dumps(payload, sort_keys=True).encode("utf-8")
self.send_response(status)
self.send_header("content-type", "application/json")
self.send_header("content-length", str(len(raw)))
self.end_headers()
self.wfile.write(raw)
def _sse(self, entry: dict[str, Any], events: list[bytes], close_after_events: int | None = None) -> None:
self.send_response(200)
self.send_header("content-type", "text/event-stream")
self.send_header("cache-control", "no-cache")
self.end_headers()
seen_stop = False
sent = 0
for event in events:
if event.startswith(b": delay-ms:"):
try:
time.sleep(int(event.decode().split(":", 2)[2]) / 1000.0)
except (IndexError, ValueError):
pass
continue
sent += 1
decoded = event.decode("utf-8", errors="replace")
event_entry = {
"raw": decoded[:500],
"after_stop": seen_stop,
"is_stop": is_stop_event(decoded),
"broken_pipe": False,
}
entry["response_events"].append(event_entry)
try:
self.wfile.write(event)
self.wfile.flush()
except BrokenPipeError:
entry["broken_pipe"] = True
event_entry["broken_pipe"] = True
return
if event_entry["is_stop"]:
seen_stop = True
if close_after_events is not None and sent >= close_after_events:
entry["closed_mid_stream"] = True
self.close_connection = True
return
def select_response(case: dict[str, Any], body: dict[str, Any], requests_before: int) -> dict[str, Any]:
if has_tool_result(body):
sequence = case.get("provider_followup_sequence")
if isinstance(sequence, list):
index = max(0, requests_before - 1)
if index < len(sequence) and isinstance(sequence[index], dict):
followup = sequence[index]
if followup.get("kind") == "error":
return followup
return {
"kind": "stream",
"steps": adapt_steps_to_tools(followup.get("steps", []), body, requests_before),
"close_after_events": followup.get("close_after_events"),
}
followup = case.get("provider_followup")
if isinstance(followup, dict):
if followup.get("kind") == "error":
return followup
if isinstance(followup.get("steps"), list):
return {
"kind": "stream",
"steps": adapt_steps_to_tools(followup["steps"], body, requests_before),
"close_after_events": followup.get("close_after_events"),
}
retry = case.get("provider_retry")
if isinstance(retry, dict):
after_requests = int(retry.get("after_requests", 1))
if requests_before >= after_requests:
if retry.get("kind") == "error":
return retry
if isinstance(retry.get("steps"), list):
return {
"kind": "stream",
"steps": adapt_steps_to_tools(retry["steps"], body, requests_before),
"close_after_events": retry.get("close_after_events"),
}
return {
"kind": "stream",
"steps": adapt_steps_to_tools(case.get("steps", []), body, requests_before),
"close_after_events": case.get("close_after_events"),
}
def render_step_templates(value: Any, request_index: int) -> Any:
if isinstance(value, str):
return (
value.replace("{{request_index}}", str(request_index))
.replace("{{request_number}}", str(request_index + 1))
)
if isinstance(value, list):
return [render_step_templates(item, request_index) for item in value]
if isinstance(value, dict):
return {key: render_step_templates(item, request_index) for key, item in value.items()}
return value
def adapt_steps_to_tools(steps: list[dict[str, Any]], body: dict[str, Any], request_index: int = 0) -> list[dict[str, Any]]:
advertised = advertised_tool_schemas(body)
if not advertised:
return [render_step_templates(step, request_index) for step in steps]
shell_tool = choose_shell_tool(advertised)
if not shell_tool:
return [render_step_templates(step, request_index) for step in steps]
shell_name, shell_arg = shell_tool
shell_call_ids: set[str] = set()
out: list[dict[str, Any]] = []
changed = False
for action, payload in normalized_steps(steps):
payload = render_step_templates(dict(payload), request_index)
if action == "provider.stream.tool_call_start":
name = str(payload.get("name", ""))
if name in {"Bash", "Shell", "bash", "shell"} and name not in advertised:
payload["name"] = shell_name
shell_call_ids.add(str(payload.get("id", "")))
changed = True
elif name == "Edit" and "Edit" not in advertised and "edit" in advertised:
payload["name"] = "edit"
changed = True
elif action == "provider.stream.tool_args_delta" and str(payload.get("id", "")) in shell_call_ids:
adapted = adapt_shell_arguments(str(payload.get("delta", "")), shell_arg, advertised.get(shell_name, set()))
if adapted != payload.get("delta"):
payload["delta"] = adapted
changed = True
out.append({action: payload})
return out if changed else steps
def advertised_tool_schemas(body: dict[str, Any]) -> dict[str, set[str]]:
schemas: dict[str, set[str]] = {}
raw_tools = body.get("tools")
if not isinstance(raw_tools, list):
return schemas
for item in raw_tools:
if not isinstance(item, dict):
continue
function = item.get("function")
if isinstance(function, dict):
name = str(function.get("name", ""))
params = function.get("parameters", {})
else:
name = str(item.get("name", ""))
params = item.get("parameters") or item.get("input_schema") or {}
properties = params.get("properties", {}) if isinstance(params, dict) else {}
if name:
schemas[name] = set(str(key) for key in properties) if isinstance(properties, dict) else set()
return schemas
def choose_shell_tool(advertised: dict[str, set[str]]) -> tuple[str, str] | None:
candidates = (
("exec_command", "cmd"),
("run_shell_command", "command"),
("run_commands", "commands"),
("terminal", "command"),
("run_command", "command"),
("shell", "command"),
("bash", "command"),
("Bash", "command"),
)
for name, preferred_arg in candidates:
if name not in advertised:
continue
properties = advertised[name]
if preferred_arg in properties or not properties:
return name, preferred_arg
if "command" in properties:
return name, "command"
if "cmd" in properties:
return name, "cmd"
if "commands" in properties:
return name, "commands"
return None
def adapt_shell_arguments(raw: str, arg_name: str, properties: set[str]) -> str:
try:
args = json.loads(raw)
except json.JSONDecodeError:
return raw
if not isinstance(args, dict):
return raw
command = args.get("command") or args.get("cmd")
if not isinstance(command, str):
return raw
adapted = dict(args)
adapted.pop("command", None)
adapted.pop("cmd", None)
adapted[arg_name] = [command] if arg_name == "commands" else command
if "description" in properties and "description" not in adapted:
adapted["description"] = "standalone shell repro"
return json.dumps(adapted, separators=(",", ":"))
def wire_for_path(path: str) -> str:
if path.startswith("/v1/messages"):
return "anthropic"
if path == "/v1/responses":
return "openai_responses"
if path == "/v1/chat/completions":
return "openai_chat"
return "unknown"
def is_stop_event(raw: str) -> bool:
return "message_stop" in raw or "response.completed" in raw or raw.strip() == "data: [DONE]"
def encode_stream(case: dict[str, Any], steps: list[dict[str, Any]], wire: str, model: str) -> list[bytes]:
if wire == "anthropic":
return encode_anthropic(steps, model or "claude-3-5-sonnet-20241022", bool(case.get("anthropic_full_input")))
if wire == "openai_responses":
return encode_openai_responses(steps, model or "gpt-4")
if wire == "openai_chat":
return encode_openai_chat(steps, model or "gpt-4")
return [sse("error", {"error": {"message": "unsupported path"}})]
def normalized_steps(steps: list[dict[str, Any]]) -> list[tuple[str, dict[str, Any]]]:
out = []
for item in steps:
if not isinstance(item, dict) or len(item) != 1:
raise ValueError(f"invalid step: {item!r}")
action, payload = next(iter(item.items()))
out.append((str(action), payload if isinstance(payload, dict) else {}))
return out
def tool_args_by_id(steps: list[dict[str, Any]]) -> dict[str, str]:
out: dict[str, str] = {}
for action, payload in normalized_steps(steps):
if action == "provider.stream.tool_args_delta":
call_id = str(payload.get("id", ""))
out[call_id] = out.get(call_id, "") + str(payload.get("delta", ""))
return out
def encode_anthropic(steps: list[dict[str, Any]], model: str, full_input: bool = False) -> list[bytes]:
events: list[bytes] = []
args_by_id = tool_args_by_id(steps)
message_started = False
next_index = 0
open_index: int | None = None
index_by_id: dict[str, int] = {}
def message_start() -> None:
nonlocal message_started
if message_started:
return
message_started = True
events.append(
sse(
"message_start",
{
"type": "message_start",
"message": {
"id": "msg_repro",
"type": "message",
"role": "assistant",
"model": model,
"content": [],
"stop_reason": None,
"usage": {"input_tokens": 1, "output_tokens": 1},
},
},
)
)
def block_stop() -> None:
nonlocal open_index
if open_index is not None:
events.append(sse("content_block_stop", {"type": "content_block_stop", "index": open_index}))
open_index = None
for action, payload in normalized_steps(steps):
if action == "provider.stream.message_start":
message_start()
elif action == "provider.stream.tool_call_start":
message_start()
block_stop()
call_id = str(payload["id"])
name = str(payload["name"])
open_index = next_index
index_by_id[call_id] = open_index
next_index += 1
events.append(
sse(
"content_block_start",
{
"type": "content_block_start",
"index": open_index,
"content_block": {
"type": "tool_use",
"id": call_id,
"name": name,
"input": parse_json_object(args_by_id.get(call_id, "")) if full_input else {},
},
},
)
)
elif action == "provider.stream.tool_args_delta":
call_id = str(payload.get("id", ""))
if full_input and call_id in index_by_id:
continue
message_start()
events.append(
sse(
"content_block_delta",
{
"type": "content_block_delta",
"index": index_by_id.get(call_id, open_index or 0),
"delta": {"type": "input_json_delta", "partial_json": str(payload.get("delta", ""))},
},
)
)
elif action == "provider.stream.tool_call_stop":
block_stop()
elif action == "provider.stream.message_stop":
block_stop()
events.append(sse("message_stop", {"type": "message_stop"}))
elif action == "provider.time.delay_ms":
events.append(f": delay-ms:{int(payload.get('value', 0))}\n\n".encode())
if message_started and not any(b"message_stop" in event for event in events):
block_stop()
events.append(sse("message_stop", {"type": "message_stop"}))
return events
def encode_openai_responses(steps: list[dict[str, Any]], model: str) -> list[bytes]:
response_id = "resp_repro"
events = [
sse(
"response.created",
{
"type": "response.created",
"response": {
"id": response_id,
"object": "response",
"created_at": int(time.time()),
"status": "in_progress",
"model": model,
"output": [],
},
},
),
sse("response.in_progress", {"type": "response.in_progress", "response": {"id": response_id, "status": "in_progress"}}),
]
output_index = 0
item_by_id: dict[str, dict[str, Any]] = {}
args_by_id: dict[str, str] = {}
for action, payload in normalized_steps(steps):
if action == "provider.stream.tool_call_start":
call_id = str(payload["id"])
item = {
"id": "fc_" + call_id,
"type": "function_call",
"call_id": call_id,
"name": str(payload["name"]),
"arguments": "",
"status": "in_progress",
}
item_by_id[call_id] = item
args_by_id.setdefault(call_id, "")
events.append(
sse(
"response.output_item.added",
{
"type": "response.output_item.added",
"output_index": output_index,
"item": item,
},
)
)
elif action == "provider.stream.tool_args_delta":
call_id = str(payload.get("id", ""))
delta = str(payload.get("delta", ""))
args_by_id[call_id] = args_by_id.get(call_id, "") + delta
events.append(
sse(
"response.function_call_arguments.delta",
{
"type": "response.function_call_arguments.delta",
"output_index": output_index,
"delta": delta,
},
)
)
elif action == "provider.stream.tool_call_stop":
call_id = str(payload.get("id", ""))
item = dict(item_by_id.get(call_id, {}))
if item:
item["status"] = "completed"
item["arguments"] = args_by_id.get(call_id, "")
item_by_id[call_id] = item
events.append(
sse(
"response.function_call_arguments.done",
{
"type": "response.function_call_arguments.done",
"output_index": output_index,
"arguments": args_by_id.get(call_id, ""),
},
)
)
events.append(
sse(
"response.output_item.done",
{
"type": "response.output_item.done",
"output_index": output_index,
"item": item,
},
)
)
output_index += 1
elif action == "provider.stream.message_stop":
events.append(
sse(
"response.completed",
{
"type": "response.completed",
"response": {
"id": response_id,
"object": "response",
"status": "completed",
"model": model,
"output": list(item_by_id.values()),
"usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2},
},
},
)
)
elif action == "provider.time.delay_ms":
events.append(f": delay-ms:{int(payload.get('value', 0))}\n\n".encode())
if not any(b"response.completed" in event for event in events):
events.append(
sse(
"response.completed",
{
"type": "response.completed",
"response": {
"id": response_id,
"object": "response",
"status": "completed",
"model": model,
"output": list(item_by_id.values()),
"usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2},
},
},
)
)
return events
def encode_openai_chat(steps: list[dict[str, Any]], model: str) -> list[bytes]:
events: list[bytes] = []
chat_id = "chatcmpl_repro"
started = False
next_index = 0
index_by_id: dict[str, int] = {}
def start() -> None:
nonlocal started
if started:
return
started = True
events.append(
sse(
"",
{
"id": chat_id,
"object": "chat.completion.chunk",
"created": int(time.time()),
"model": model,
"choices": [{"index": 0, "delta": {"role": "assistant"}, "finish_reason": None}],
},
)
)
has_tool_calls = False
for action, payload in normalized_steps(steps):
if action == "provider.stream.tool_call_start":
start()
has_tool_calls = True
call_id = str(payload["id"])
index_by_id[call_id] = next_index
next_index += 1
events.append(
sse(
"",
{
"choices": [
{
"index": 0,
"delta": {
"tool_calls": [
{
"index": index_by_id[call_id],
"id": call_id,
"type": "function",
"function": {"name": str(payload["name"]), "arguments": ""},
}
]
},
"finish_reason": None,
}
]
},
)
)
elif action == "provider.stream.tool_args_delta":
start()
call_id = str(payload.get("id", ""))
events.append(
sse(
"",
{
"choices": [
{
"index": 0,
"delta": {
"tool_calls": [
{
"index": index_by_id.get(call_id, 0),
"function": {"arguments": str(payload.get("delta", ""))},
}
]
},
"finish_reason": None,
}
]
},
)
)
elif action == "provider.stream.message_stop":
start()
events.append(
sse(
"",
{
"id": chat_id,
"object": "chat.completion.chunk",
"created": int(time.time()),
"model": model,
"choices": [
{
"index": 0,
"delta": {},
"finish_reason": "tool_calls" if has_tool_calls else "stop",
}
],
},
)
)
events.append(b"data: [DONE]\n\n")
elif action == "provider.time.delay_ms":
events.append(f": delay-ms:{int(payload.get('value', 0))}\n\n".encode())
if not events or events[-1] != b"data: [DONE]\n\n":
start()
events.append(
sse(
"",
{
"id": chat_id,
"object": "chat.completion.chunk",
"created": int(time.time()),
"model": model,
"choices": [
{
"index": 0,
"delta": {},
"finish_reason": "tool_calls" if has_tool_calls else "stop",
}
],
},
)
)
events.append(b"data: [DONE]\n\n")
return events
def parse_json_object(value: str) -> dict[str, Any]:
try:
parsed = json.loads(value or "{}")
except json.JSONDecodeError:
return {}
return parsed if isinstance(parsed, dict) else {}
def sse(event: str, payload: dict[str, Any]) -> bytes:
prefix = f"event: {event}\n" if event else ""
return (prefix + "data: " + json.dumps(payload, sort_keys=True) + "\n\n").encode()
def has_tool_result(value: Any) -> bool:
if isinstance(value, list):
return any(has_tool_result(item) for item in value)
if not isinstance(value, dict):
return False
raw_type = str(value.get("type", "")).lower()
role = str(value.get("role", "")).lower()
if raw_type in {"function_call_output", "tool_result", "tool_use_result", "tool_call_output"}:
return True
if role == "tool":
return True
return any(has_tool_result(item) for item in value.values())
def collect_tool_results(value: Any, out: list[dict[str, Any]]) -> None:
if isinstance(value, list):
for item in value:
collect_tool_results(item, out)
return
if not isinstance(value, dict):
return
raw_type = str(value.get("type", "")).lower()
role = str(value.get("role", "")).lower()
result_like = raw_type in {"function_call_output", "tool_result", "tool_use_result", "tool_call_output"} or role == "tool"
if result_like:
call_id = ""
for key in ("call_id", "tool_call_id", "tool_use_id"):
if value.get(key):
call_id = str(value[key])
break
if call_id:
payload = value.get("output") if "output" in value else value.get("result") if "result" in value else value.get("content")
out.append(
{
"call_id": call_id,
"type": raw_type or role or "tool_result",
"payload_text": json.dumps(payload, default=str, sort_keys=True),
}
)
for item in value.values():
collect_tool_results(item, out)
def build_image(case: dict[str, Any], image: str) -> None:
dockerfile = str(
case.get("dockerfile")
or dockerfile_for_target(str(case["target"]), str(case.get("version", image.rsplit(":", 1)[-1])))
)
with tempfile.TemporaryDirectory(prefix="vendor-repro-build-") as tmp:
path = Path(tmp) / "Dockerfile"
path.write_text(dockerfile, encoding="utf-8")
cmd = [
"docker",
"build",
"--build-arg",
f"RUNNER_UID={os.getuid()}",
"--build-arg",
f"RUNNER_GID={os.getgid()}",
"-t",
image,
"-f",
str(path),
tmp,
]
run_checked(cmd)
def run_checked(cmd: list[str]) -> None:
print("+ " + " ".join(cmd), flush=True)
proc = subprocess.run(cmd, text=True)
if proc.returncode != 0:
raise SystemExit(proc.returncode)
def derive_image(case: dict[str, Any]) -> str:
if case.get("image"):
return str(case["image"])
return f"vendor-repro-{case['target']}:{case.get('version', 'latest')}"
def target_command(target: str, base_url: str, workspace: str, timeout_s: int) -> tuple[list[str], dict[str, str]]:
if target == "claude":
return (
[
"claude",
"-p",
PROMPT,
"--dangerously-skip-permissions",
"--output-format",
"text",
"--model",
"claude-3-5-sonnet-20241022",
],
{
"ANTHROPIC_BASE_URL": base_url,
"ANTHROPIC_API_KEY": "vendor-repro-fake-key",
"ANTHROPIC_AUTH_TOKEN": "vendor-repro-fake-key",
"DISABLE_AUTOUPDATER": "1",
"CLAUDE_CODE_DISABLE_TELEMETRY": "1",
"DISABLE_TELEMETRY": "1",
"DISABLE_ERROR_REPORTING": "1",
"DISABLE_NON_ESSENTIAL_MODEL_CALLS": "1",
},
)
if target == "codex":
base = base_url.rstrip("/")
return (
[
"codex",
"exec",
"-c",
'model_provider="vendor-repro"',
"-c",
'model_providers.vendor-repro.name="vendor-repro"',
"-c",
f'model_providers.vendor-repro.base_url="{base}/v1"',
"-c",
'model_providers.vendor-repro.wire_api="responses"',
"-c",
"model_providers.vendor-repro.requires_openai_auth=false",
"-c",
'approval_policy="never"',
"-c",
'sandbox_mode="danger-full-access"',
"-c",
'model="gpt-4"',
"-c",
'model_reasoning_effort="low"',
"--skip-git-repo-check",
PROMPT,
],
{"OPENAI_API_KEY": "vendor-repro-fake-key"},
)
if target == "goose":
return (
[
"goose",
"run",
"--no-profile",
"--with-builtin",
"developer",
"--max-turns",
str(max(3, timeout_s // 10)),
"--output-format",
"json",
"--text",
PROMPT,
],
{
"ANTHROPIC_API_KEY": "vendor-repro-fake-key",
"ANTHROPIC_HOST": base_url.rstrip(),
"GOOSE_PROVIDER": "anthropic",
"GOOSE_MODEL": "claude-sonnet-4-5",
},
)
if target == "crush":
return (
[
"crush",
"--cwd",
workspace,
"--data-dir",
workspace + "/vendor_repro_crush_home/.local/share/crush",
"run",
"--quiet",
"-m",
"local-repro/gpt-4",
PROMPT,
],
{"OPENAI_API_KEY": "vendor-repro-fake-key"},
)
if target == "qwen":
return (
[
"/home/runner/.npm-global/bin/qwen",
"--chat-recording=false",
"--auth-type",
"openai",
"--openai-api-key",
"vendor-repro-fake-key",
"--openai-base-url",
base_url.rstrip("/") + "/v1",
"--model",
"gpt-4",
"--approval-mode",
"yolo",
"--output-format",
"text",
"-p",
PROMPT,
],
{"OPENAI_API_KEY": "vendor-repro-fake-key", "QWEN_CODE_SUPPRESS_YOLO_WARNING": "1"},
)
if target == "cline":
base = base_url.rstrip("/") + "/v1"
data_dir = "/workspace/vendor_repro_cline_data"
model = "gpt-4"
auth_cmd = " ".join(
[
shlex.quote("cline"),
"auth",
"--provider",
"openai-native",
"--apikey",
"vendor-repro-fake-key",
"--modelid",
shlex.quote(model),
"--baseurl",
shlex.quote(base),
"--data-dir",
shlex.quote(data_dir),
]
)
run_cmd = " ".join(
[
"exec",
shlex.quote("cline"),
"--data-dir",
shlex.quote(data_dir),
"--cwd",
shlex.quote(workspace),
"-P",
"openai-native",
"-m",
shlex.quote(model),
"-k",
"vendor-repro-fake-key",
"--json",
"--retries",
"100",
"-y",
shlex.quote(PROMPT),
]
)
return (
["bash", "-c", auth_cmd + " && " + run_cmd],
{
"CLINE_DATA_DIR": data_dir,
"CLINE_LOG_ENABLED": "0",
"OPENAI_API_KEY": "vendor-repro-fake-key",
},
)
if target == "opencode":
return (
["opencode", "run", "-m", "anthropic/claude-haiku-4-5-20251001", PROMPT],
{
"ANTHROPIC_API_KEY": "vendor-repro-fake-key",
"ANTHROPIC_BASE_URL": base_url.rstrip("/") + "/v1",
"OPENCODE_DISABLE_DEFAULT_PLUGINS": "1",
},
)
if target == "continue":
return (
[
"cn",
"--config",
"/workspace/vendor_repro_continue_home/config.yaml",
"--auto",
"--silent",
"-p",
PROMPT,
],
{
"CONTINUE_HOME": "/workspace/vendor_repro_continue_home",
"OPENAI_API_KEY": "vendor-repro-fake-key",
},
)
raise ValueError(f"unsupported target: {target}")
def prepare_workspace(root: Path, target: str, base_url: str, case: dict[str, Any]) -> dict[str, str]:
root.mkdir(parents=True, exist_ok=True)
(root / "README.md").write_text("# Vendor repro workspace\n", encoding="utf-8")
files = case.get("workspace_files", {})
if isinstance(files, dict):
for raw_path, content in files.items():
rel = Path(str(raw_path))
if rel.is_absolute() or ".." in rel.parts:
raise ValueError(f"unsafe workspace path: {raw_path}")
destination = root / rel
destination.parent.mkdir(parents=True, exist_ok=True)
destination.write_text(str(content), encoding="utf-8")
env: dict[str, str] = {}
if target == "crush":
host_home = root / "vendor_repro_crush_home"
host_config_dir = host_home / ".config" / "crush"
host_data_dir = host_home / ".local" / "share" / "crush"
host_config_dir.mkdir(parents=True, exist_ok=True)
host_data_dir.mkdir(parents=True, exist_ok=True)
(host_config_dir / "crush.json").write_text(
json.dumps(
{
"$schema": "https://charm.land/crush.json",
"providers": {
"local-repro": {
"type": "openai-compat",
"base_url": base_url.rstrip("/") + "/v1",
"api_key": "vendor-repro-fake-key",
"models": [
{
"id": "gpt-4",
"name": "Local repro gpt-4",
"context_window": 128000,
"default_max_tokens": 2000,
}
],
}
},
},
sort_keys=True,
),
encoding="utf-8",
)
env.update(
{
"HOME": "/workspace/vendor_repro_crush_home",
"XDG_CONFIG_HOME": "/workspace/vendor_repro_crush_home/.config",
"XDG_DATA_HOME": "/workspace/vendor_repro_crush_home/.local/share",
"CRUSH_GLOBAL_CONFIG": "/workspace/vendor_repro_crush_home/.config/crush",
"CRUSH_GLOBAL_DATA": "/workspace/vendor_repro_crush_home/.local/share/crush",
}
)
if target == "goose":
home = root / "vendor_repro_goose_home"
for rel in (".config", ".cache", ".local/share", ".local/state"):
(home / rel).mkdir(parents=True, exist_ok=True)
env.update(
{
"HOME": "/workspace/vendor_repro_goose_home",
"XDG_CONFIG_HOME": "/workspace/vendor_repro_goose_home/.config",
"XDG_CACHE_HOME": "/workspace/vendor_repro_goose_home/.cache",
"XDG_DATA_HOME": "/workspace/vendor_repro_goose_home/.local/share",
"XDG_STATE_HOME": "/workspace/vendor_repro_goose_home/.local/state",
}
)
if target == "qwen":
host_home = root / "vendor_repro_qwen_home"
host_config = host_home / ".qwen"
host_config.mkdir(parents=True, exist_ok=True)
(host_config / "settings.json").write_text(
json.dumps(
{
"modelProviders": {
"openai": [
{
"id": "gpt-4",
"name": "gpt-4",
"baseUrl": base_url.rstrip("/") + "/v1",
"envKey": "OPENAI_API_KEY",
}
]
},
"env": {"OPENAI_API_KEY": "vendor-repro-fake-key"},
"security": {"auth": {"selectedType": "openai"}},
"model": {"name": "gpt-4"},
"telemetry": {"enabled": False},
},
sort_keys=True,
),
encoding="utf-8",
)
env["HOME"] = "/workspace/vendor_repro_qwen_home"
if target == "cline":
host_data = root / "vendor_repro_cline_data"
host_data.mkdir(parents=True, exist_ok=True)
host_data.chmod(0o777)
env["CLINE_DATA_DIR"] = "/workspace/vendor_repro_cline_data"
if target == "continue":
host_home = root / "vendor_repro_continue_home"
host_home.mkdir(parents=True, exist_ok=True)
config_path = host_home / "config.yaml"
config_path.write_text(
"name: Vendor Reproducer\n"
"version: 1.0.0\n"
"schema: v1\n"
"models:\n"
" - name: Vendor GPT-4\n"
" provider: openai\n"
" model: gpt-4\n"
" apiKey: vendor-repro-fake-key\n"
" apiBase: " + base_url.rstrip("/") + "/v1\n",
encoding="utf-8",
)
config_path.chmod(0o666)
env["CONTINUE_HOME"] = "/workspace/vendor_repro_continue_home"
if target == "opencode":
host_home = root / "vendor_repro_opencode_home"
for rel in (".config", ".cache", ".local/share", ".local/share/opencode"):
(host_home / rel).mkdir(parents=True, exist_ok=True)
env.update(
{
"HOME": "/workspace/vendor_repro_opencode_home",
"XDG_CONFIG_HOME": "/workspace/vendor_repro_opencode_home/.config",
"XDG_CACHE_HOME": "/workspace/vendor_repro_opencode_home/.cache",
"XDG_DATA_HOME": "/workspace/vendor_repro_opencode_home/.local/share",
"OPENCODE_DISABLE_DEFAULT_PLUGINS": "1",
}
)
if case.get("side_effect_probe"):
env["REPRO_SIDE_EFFECT_ROOT"] = "/side-effects"
return env
def run_target(
case: dict[str, Any],
image: str,
base_url: str,
workspace: Path,
side_effects: Path,
extra_env: dict[str, str],
) -> subprocess.CompletedProcess[bytes]:
target = str(case["target"])
timeout_s = int(case.get("timeout_s", 90))
inner, env = target_command(target, base_url, "/workspace", timeout_s)
env = {**extra_env, **env}
cmd = [
"docker",
"run",
"--rm",
"-v",
f"{workspace}:/workspace:rw",
"-v",
f"{side_effects}:/side-effects:rw",
"-w",
"/workspace",
*docker_network_args(base_url),
"--cpus",
str(case.get("docker_cpus", DEFAULT_DOCKER_CPUS)),
"--memory",
str(case.get("docker_memory", DEFAULT_DOCKER_MEMORY)),
"-e",
f"PATH={DOCKER_PATH}",
]
for key, value in sorted(env.items()):
cmd.extend(["-e", f"{key}={value}"])
docker_user = os.environ.get("MODELLOCK_REPRO_DOCKER_USER")
if docker_user is None and target == "claude":
docker_user = f"{os.getuid()}:{os.getgid()}"
cmd.extend(["--entrypoint", "/usr/bin/env"])
if docker_user:
cmd.extend(["--user", docker_user])
cmd.append(image)
if case.get("side_effect_probe"):
pid_file = str(case.get("pid_file", "child.pid"))
alive_file = str(case.get("alive_file", "child.alive"))
quoted_inner = " ".join(shlex.quote(part) for part in inner)
probe = (
"status=0; "
+ quoted_inner
+ " || status=$?; "
+ "python3 - <<'PY'\n"
+ "import os, pathlib\n"
+ "root = pathlib.Path('/side-effects')\n"
+ f"pid_path = root / {pid_file!r}\n"
+ f"alive_path = root / {alive_file!r}\n"
+ "alive = False\n"
+ "pid_text = ''\n"
+ "if pid_path.exists():\n"
+ " pid_text = pid_path.read_text().strip()\n"
+ " try:\n"
+ " os.kill(int(pid_text), 0)\n"
+ " alive = True\n"
+ " except Exception:\n"
+ " alive = False\n"
+ "alive_path.write_text(('alive' if alive else 'dead') + '\\n' + pid_text)\n"
+ "PY\n"
+ "exit $status"
)
cmd.extend(["bash", "-c", probe])
else:
cmd.extend(inner)
print("+ " + " ".join(cmd), flush=True)
proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, start_new_session=True)
signal_after = case.get("signal_after_ms")
if signal_after is not None:
def send_signal() -> None:
time.sleep(float(signal_after) / 1000.0)
try:
os.killpg(os.getpgid(proc.pid), signal.SIGINT)
except ProcessLookupError:
pass
threading.Thread(target=send_signal, daemon=True).start()
try:
stdout, stderr = proc.communicate(timeout=timeout_s)
result = subprocess.CompletedProcess(cmd, proc.returncode, stdout, stderr)
except subprocess.TimeoutExpired:
try:
os.killpg(os.getpgid(proc.pid), signal.SIGTERM)
except ProcessLookupError:
pass
try:
stdout, stderr = proc.communicate(timeout=5)
except subprocess.TimeoutExpired:
try:
os.killpg(os.getpgid(proc.pid), signal.SIGKILL)
except ProcessLookupError:
pass
stdout, stderr = proc.communicate()
result = subprocess.CompletedProcess(cmd, proc.returncode, stdout, stderr)
result.timed_out = True # type: ignore[attr-defined]
return result
result.timed_out = False # type: ignore[attr-defined]
return result
def provider_host_for_container() -> str:
override = os.environ.get("MODELLOCK_REPRO_PROVIDER_HOST")
if override:
return override
if sys.platform.startswith("linux"):
return "127.0.0.1"
return "host.docker.internal"
def docker_network_args(base_url: str) -> list[str]:
if "127.0.0.1" in base_url or "localhost" in base_url:
return ["--network", "host"]
if "host.docker.internal" in base_url:
return ["--add-host", "host.docker.internal:host-gateway"]
return []
def temporary_workspace() -> Any:
base = Path(os.environ.get("MODELLOCK_REPRO_TMPDIR", Path.cwd())).resolve()
base.mkdir(parents=True, exist_ok=True)
return _BestEffortTemporaryDirectory(prefix="vendor-repro-workspace-", dir=str(base))
class _BestEffortTemporaryDirectory:
def __init__(self, prefix: str, dir: str) -> None:
self.name = tempfile.mkdtemp(prefix=prefix, dir=dir)
def __enter__(self) -> str:
return self.name
def __exit__(self, exc_type: object, exc: object, tb: object) -> None:
try:
shutil.rmtree(self.name)
except Exception:
pass
def expanded_case_runs(case: dict[str, Any]) -> list[dict[str, Any]]:
runs = case.get("runs")
if not isinstance(runs, list):
return [case]
base = {key: value for key, value in case.items() if key != "runs"}
expanded: list[dict[str, Any]] = []
for index, run in enumerate(runs, 1):
if not isinstance(run, dict):
raise ValueError(f"invalid run entry: {run!r}")
merged = {**base, **run}
merged.setdefault("run_name", f"run{index}")
expanded.append(merged)
return expanded
def is_pid_alive(pid: int) -> bool:
if pid <= 0:
return False
try:
os.kill(pid, 0)
except ProcessLookupError:
return False
except PermissionError:
return True
return True
def response_events(requests: list[dict[str, Any]]) -> list[dict[str, Any]]:
events: list[dict[str, Any]] = []
for request in requests:
for event in request.get("response_events", []):
if isinstance(event, dict):
events.append(event)
return events
def max_tool_result_count_in_one_request(requests: list[dict[str, Any]], call_id: str = "") -> int:
max_count = 0
for request in requests:
results: list[dict[str, Any]] = []
collect_tool_results(request.get("body"), results)
counts: dict[str, int] = {}
for result in results:
current = str(result.get("call_id", ""))
if call_id and current != call_id:
continue
counts[current] = counts.get(current, 0) + 1
if counts:
max_count = max(max_count, max(counts.values()))
return max_count
def evaluate(
case: dict[str, Any],
requests: list[dict[str, Any]],
proc: subprocess.CompletedProcess[bytes],
side_effects: Path,
) -> tuple[bool, list[str]]:
results: list[dict[str, Any]] = []
for request in requests:
collect_tool_results(request.get("body"), results)
result_ids = [str(result["call_id"]) for result in results]
counts: dict[str, int] = {}
for call_id in result_ids:
counts[call_id] = counts.get(call_id, 0) + 1
events = response_events(requests)
late_events = [
event
for event in events
if event.get("after_stop") and not event.get("broken_pipe") and ("delta" in str(event.get("raw", "")))
]
total_result_bytes = sum(len(str(result.get("payload_text", "")).encode("utf-8", errors="ignore")) for result in results)
closed_mid_stream = sum(1 for request in requests if request.get("closed_mid_stream"))
timed_out = bool(getattr(proc, "timed_out", False))
checks = case.get("checks", [])
messages = [
f"process_exit={proc.returncode}",
f"process_timed_out={timed_out}",
f"provider_requests={len(requests)}",
f"tool_result_counts={dict(sorted(counts.items()))}",
f"tool_result_bytes={total_result_bytes}",
f"closed_mid_stream={closed_mid_stream}",
f"late_stream_events_after_stop={len(late_events)}",
]
ok = True
for check in checks:
kind = check.get("kind")
call_id = str(check.get("call_id", ""))
if kind == "duplicate_tool_result":
minimum = int(check.get("min_count", 2))
actual = counts.get(call_id, 0)
passed = actual >= minimum
messages.append(f"duplicate_tool_result {call_id}: observed={actual} expected>={minimum}")
ok = ok and passed
elif kind == "duplicate_tool_result_in_request":
minimum = int(check.get("min_count", 2))
actual = max_tool_result_count_in_one_request(requests, call_id)
passed = actual >= minimum
messages.append(f"duplicate_tool_result_in_request {call_id or '<any>'}: observed={actual} expected>={minimum}")
ok = ok and passed
elif kind == "tool_result":
actual = counts.get(call_id, 0)
passed = actual >= int(check.get("min_count", 1))
messages.append(f"tool_result {call_id}: observed={actual} expected>={int(check.get('min_count', 1))}")
ok = ok and passed
elif kind == "tool_result_payload_contains":
needle = str(check.get("text", ""))
actual = any(result["call_id"] == call_id and needle in result.get("payload_text", "") for result in results)
messages.append(f"tool_result_payload_contains {call_id} {needle!r}: observed={actual}")
ok = ok and actual
elif kind == "any_tool_result_after_signal":
minimum = int(check.get("min_count", 1))
actual = len(result_ids)
passed = actual >= minimum
messages.append(f"tool_result_after_signal: observed={actual} expected>={minimum}")
ok = ok and passed
elif kind == "late_stream_events_after_stop":
minimum = int(check.get("min_count", 1))
actual = len(late_events)
passed = actual >= minimum
messages.append(f"late_stream_events_after_stop: observed={actual} expected>={minimum}")
ok = ok and passed
elif kind == "pid_file_alive":
raw_path = str(check.get("path", ""))
rel = Path(raw_path)
if rel.is_absolute() or ".." in rel.parts:
raise ValueError(f"unsafe side-effect path: {raw_path}")
pid_file = side_effects / rel
pid_text = pid_file.read_text(encoding="utf-8").strip() if pid_file.exists() else ""
alive_file = side_effects / str(check.get("alive_file", case.get("alive_file", "child.alive")))
if alive_file.exists():
alive_text = alive_file.read_text(encoding="utf-8").strip().splitlines()
actual = bool(alive_text and alive_text[0] == "alive")
if len(alive_text) > 1 and not pid_text:
pid_text = alive_text[1]
else:
try:
pid = int(pid_text)
except ValueError:
pid = -1
actual = is_pid_alive(pid)
messages.append(f"pid_file_alive {raw_path}: pid={pid_text or '<missing>'} observed={actual}")
ok = ok and actual
elif kind == "side_effect_file_contains":
raw_path = str(check.get("path", ""))
rel = Path(raw_path)
if rel.is_absolute() or ".." in rel.parts:
raise ValueError(f"unsafe side-effect path: {raw_path}")
needle = str(check.get("text", ""))
path = side_effects / rel
content = path.read_text(encoding="utf-8", errors="replace") if path.exists() else ""
actual = needle in content
messages.append(f"side_effect_file_contains {raw_path} {needle!r}: observed={actual}")
ok = ok and actual
elif kind == "side_effect_file_occurrences":
raw_path = str(check.get("path", ""))
rel = Path(raw_path)
if rel.is_absolute() or ".." in rel.parts:
raise ValueError(f"unsafe side-effect path: {raw_path}")
needle = str(check.get("text", ""))
minimum = int(check.get("min_count", 1))
path = side_effects / rel
content = path.read_text(encoding="utf-8", errors="replace") if path.exists() else ""
actual = content.count(needle) if needle else 0
passed = actual >= minimum
messages.append(f"side_effect_file_occurrences {raw_path} {needle!r}: observed={actual} expected>={minimum}")
ok = ok and passed
elif kind == "provider_requests_at_least":
minimum = int(check.get("min_count", 1))
actual = len(requests)
passed = actual >= minimum
messages.append(f"provider_requests_at_least: observed={actual} expected>={minimum}")
ok = ok and passed
elif kind == "tool_result_bytes_at_least":
minimum = int(check.get("min_bytes", 1))
passed = total_result_bytes >= minimum
messages.append(f"tool_result_bytes_at_least: observed={total_result_bytes} expected>={minimum}")
ok = ok and passed
elif kind == "large_tool_results_at_least":
threshold = int(check.get("min_bytes", 200_000))
minimum = int(check.get("min_count", 1))
actual = sum(
1
for result in results
if len(str(result.get("payload_text", "")).encode("utf-8", errors="ignore")) >= threshold
)
passed = actual >= minimum
messages.append(f"large_tool_results_at_least: observed={actual} expected>={minimum}")
ok = ok and passed
elif kind == "closed_mid_stream_at_least":
minimum = int(check.get("min_count", 1))
passed = closed_mid_stream >= minimum
messages.append(f"closed_mid_stream_at_least: observed={closed_mid_stream} expected>={minimum}")
ok = ok and passed
elif kind == "nonzero_or_timeout":
passed = bool(proc.returncode not in (0, None) or timed_out)
messages.append(f"nonzero_or_timeout: returncode={proc.returncode} timed_out={timed_out}")
ok = ok and passed
elif kind == "timeout_or_signal":
passed = bool(timed_out or (proc.returncode is not None and proc.returncode < 0) or proc.returncode == 124)
messages.append(f"timeout_or_signal: returncode={proc.returncode} timed_out={timed_out}")
ok = ok and passed
else:
ok = False
messages.append(f"unknown check: {kind}")
return ok, messages
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument("case", type=Path, nargs="?")
parser.add_argument("--skip-build", action="store_true")
args = parser.parse_args()
if args.case is None:
if EMBEDDED_CASE is None:
parser.error("case JSON is required unless this file embeds a case")
case = EMBEDDED_CASE
else:
case = json.loads(args.case.read_text(encoding="utf-8"))
runs = expanded_case_runs(case)
if not args.skip_build:
built: set[str] = set()
for run in runs:
image = derive_image(run)
if image not in built:
build_image(run, image)
built.add(image)
all_ok = True
for run in runs:
image = derive_image(run)
state = ProviderState(run)
server = ProviderServer(("127.0.0.1", 0), state)
thread = threading.Thread(target=server.serve_forever, daemon=True)
thread.start()
_host, port = server.server_address
base_url = f"http://{provider_host_for_container()}:{port}"
try:
with temporary_workspace() as tmp:
workspace = Path(tmp)
side_effects = workspace / "side-effects"
side_effects.mkdir(parents=True, exist_ok=True)
extra_env = prepare_workspace(workspace, str(run["target"]), base_url, run)
proc = run_target(run, image, base_url, workspace, side_effects, extra_env)
requests = state.snapshot_requests()
ok, messages = evaluate(run, requests, proc, side_effects)
all_ok = all_ok and ok
print("\n=== Reproduction summary ===")
print(f"case={case.get('slug', '<embedded>')} run={run.get('run_name', run['target'])}")
print(f"target={run['target']} image={image}")
print(f"process_exit={proc.returncode}")
for message in messages:
print(message)
if proc.stdout:
print("stdout_tail=" + proc.stdout[-1000:].decode("utf-8", errors="replace"))
if proc.stderr:
print("stderr_tail=" + proc.stderr[-1000:].decode("utf-8", errors="replace"))
print("RUN_REPRODUCED" if ok else "RUN_NOT_REPRODUCED")
finally:
server.shutdown()
server.server_close()
thread.join(timeout=2)
print("REPRODUCED" if all_ok else "NOT_REPRODUCED")
return 0 if all_ok else 1
if __name__ == "__main__":
raise SystemExit(main())
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment