Created
June 13, 2026 08:02
-
-
Save N0zoM1z0/5861579c2e83fed87126293bf76bbbcf to your computer and use it in GitHub Desktop.
modelock self-contained reproducer: opencode-post-tool-transient-disconnect
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/usr/bin/env python3 | |
| """Standalone vendor reproducer for CLI provider-stream bugs. | |
| Usage: | |
| python3 reproducer.py case.json | |
| The script intentionally imports only Python standard-library modules. It | |
| starts a local mock provider, runs the target CLI in Docker, records provider | |
| requests, and evaluates simple case assertions. | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| import os | |
| from pathlib import Path | |
| import shlex | |
| import signal | |
| import subprocess | |
| import sys | |
| import tempfile | |
| import threading | |
| import time | |
| from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer | |
| from typing import Any | |
| PROMPT = "Briefly inspect this directory and use the requested tool if needed." | |
| DOCKER_PATH = "/home/runner/.npm-global/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" | |
| DEFAULT_DOCKER_CPUS = "2" | |
| DEFAULT_DOCKER_MEMORY = "4g" | |
| EMBEDDED_CASE: dict[str, Any] | None = json.loads(r''' | |
| { | |
| "checks": [ | |
| { | |
| "kind": "provider_requests_at_least", | |
| "min_count": 3 | |
| }, | |
| { | |
| "kind": "closed_mid_stream_at_least", | |
| "min_count": 1 | |
| }, | |
| { | |
| "kind": "nonzero_or_timeout" | |
| } | |
| ], | |
| "close_after_events": 4, | |
| "image": "vendor-repro-opencode:1.17.4", | |
| "steps": [ | |
| { | |
| "provider.stream.tool_call_start": { | |
| "id": "disconnect_probe", | |
| "name": "Bash" | |
| } | |
| }, | |
| { | |
| "provider.stream.tool_args_delta": { | |
| "delta": "{\"command\":\"printf MODELOCK_DISCONNECT_MID_TOOL\\\\n\"}", | |
| "id": "disconnect_probe" | |
| } | |
| }, | |
| { | |
| "provider.stream.tool_call_stop": { | |
| "id": "disconnect_probe" | |
| } | |
| }, | |
| { | |
| "provider.stream.message_stop": {} | |
| } | |
| ], | |
| "target": "opencode", | |
| "timeout_s": 75, | |
| "version": "1.17.4", | |
| "workspace_files": { | |
| "src/app.js": "console.log('modelock');\n" | |
| } | |
| } | |
| ''') | |
| def dockerfile_for_target(target: str, version: str) -> str: | |
| common = """FROM node:24-bookworm | |
| ARG RUNNER_UID=1000 | |
| ARG RUNNER_GID=1000 | |
| ENV DEBIAN_FRONTEND=noninteractive | |
| ENV NPM_CONFIG_PREFIX=/home/runner/.npm-global | |
| ENV PATH=/home/runner/.npm-global/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin | |
| RUN apt-get update && apt-get install -y --no-install-recommends \\ | |
| python3 git curl ca-certificates procps lsof ripgrep passwd \\ | |
| && rm -rf /var/lib/apt/lists/* | |
| RUN set -eux; \\ | |
| if ! getent group "${RUNNER_GID}" >/dev/null; then groupadd -g "${RUNNER_GID}" runner; fi; \\ | |
| if ! getent passwd "${RUNNER_UID}" >/dev/null; then \\ | |
| useradd -m -u "${RUNNER_UID}" -g "${RUNNER_GID}" -d /home/runner -s /bin/bash runner; \\ | |
| else \\ | |
| existing_user="$(getent passwd "${RUNNER_UID}" | cut -d: -f1)"; \\ | |
| usermod -d /home/runner "${existing_user}" || true; \\ | |
| fi; \\ | |
| mkdir -p /home/runner/.npm-global; \\ | |
| chown -R "${RUNNER_UID}:${RUNNER_GID}" /home/runner; \\ | |
| chmod -R a+rx /home/runner | |
| """ | |
| if target == "claude": | |
| install = f"RUN npm install -g @anthropic-ai/claude-code@{version} && chmod -R a+rx /home/runner\n" | |
| elif target == "codex": | |
| install = f"RUN npm install -g @openai/codex@{version} && chmod -R a+rx /home/runner\n" | |
| elif target == "qwen": | |
| install = f"RUN npm install -g @qwen-code/qwen-code@{version} && chmod -R a+rx /home/runner\n" | |
| elif target == "crush": | |
| install = f"RUN npm install -g @charmland/crush@{version} && chmod -R a+rx /home/runner\n" | |
| elif target == "goose": | |
| install = ( | |
| "RUN curl -fsSL --retry 3 --connect-timeout 20 --max-time 180 " | |
| "https://github.com/aaif-goose/goose/releases/download/stable/download_cli.sh " | |
| f"| GOOSE_VERSION=v{version} CONFIGURE=false GOOSE_BIN_DIR=/usr/local/bin bash\n" | |
| ) | |
| elif target == "cline": | |
| install = ( | |
| f"RUN npm install -g cline@{version} " | |
| f"&& cline_root=\"$(npm root -g)/@cline/cli-linux-x64\" " | |
| "&& if [ ! -x \"$cline_root/bin/cline\" ]; then " | |
| "tmp=\"$(mktemp -d)\"; " | |
| f"curl -fsSL --retry 3 https://registry.npmjs.org/@cline/cli-linux-x64/-/cli-linux-x64-{version}.tgz -o \"$tmp/cline.tgz\"; " | |
| "tar -xzf \"$tmp/cline.tgz\" -C \"$tmp\"; " | |
| "mkdir -p \"$cline_root/bin\"; " | |
| "cp \"$tmp/package/bin/cline\" \"$cline_root/bin/cline\"; " | |
| "chmod +x \"$cline_root/bin/cline\"; " | |
| "fi " | |
| "&& chmod -R a+rx /home/runner\n" | |
| ) | |
| elif target == "opencode": | |
| install = f"RUN npm install -g opencode-ai@{version} && chmod -R a+rx /home/runner\n" | |
| elif target == "continue": | |
| install = f"RUN npm install -g @continuedev/cli@{version} && chmod -R a+rx /home/runner\n" | |
| else: | |
| raise ValueError(f"unsupported target for Dockerfile: {target}") | |
| return common + install + 'WORKDIR /workspace\nENTRYPOINT ["/usr/bin/env"]\nCMD ["bash"]\n' | |
| class ProviderState: | |
| def __init__(self, case: dict[str, Any]) -> None: | |
| self.case = case | |
| self.requests: list[dict[str, Any]] = [] | |
| self.lock = threading.Lock() | |
| def append_request(self, entry: dict[str, Any]) -> None: | |
| with self.lock: | |
| self.requests.append(entry) | |
| def request_count(self) -> int: | |
| with self.lock: | |
| return len(self.requests) | |
| def snapshot_requests(self) -> list[dict[str, Any]]: | |
| with self.lock: | |
| return list(self.requests) | |
| class ProviderServer(ThreadingHTTPServer): | |
| def __init__(self, address: tuple[str, int], state: ProviderState) -> None: | |
| self.state = state | |
| super().__init__(address, ProviderHandler) | |
| class ProviderHandler(BaseHTTPRequestHandler): | |
| server: ProviderServer | |
| def log_message(self, fmt: str, *args: Any) -> None: | |
| return None | |
| def do_GET(self) -> None: | |
| if self.path in {"/health", "/__repro/health"}: | |
| self._json(200, {"ok": True}) | |
| return | |
| if self.path in {"/v1/models", "/v1/anthropic/models"}: | |
| self._json( | |
| 200, | |
| { | |
| "object": "list", | |
| "data": [ | |
| {"id": "gpt-4", "object": "model"}, | |
| {"id": "claude-3-5-sonnet-20241022", "type": "model"}, | |
| {"id": "claude-sonnet-4-5", "type": "model"}, | |
| ], | |
| }, | |
| ) | |
| return | |
| self._json(404, {"error": "not found"}) | |
| def do_POST(self) -> None: | |
| started = time.time() | |
| body = self._read_json() | |
| requests_before = self.server.state.request_count() | |
| response = select_response(self.server.state.case, body, requests_before) | |
| entry = { | |
| "ts": started, | |
| "path": self.path, | |
| "body": body, | |
| "response_events": [], | |
| "status": response.get("status", 200), | |
| "closed_mid_stream": False, | |
| } | |
| self.server.state.append_request(entry) | |
| if response.get("kind") == "error": | |
| self._json(int(response.get("status", 500)), {"error": response.get("error", {})}) | |
| return | |
| wire = wire_for_path(self.path) | |
| events = encode_stream(self.server.state.case, response.get("steps", []), wire, str(body.get("model", ""))) | |
| close_after = response.get("close_after_events") | |
| self._sse(entry, events, int(close_after) if close_after is not None else None) | |
| def _read_json(self) -> dict[str, Any]: | |
| length = int(self.headers.get("content-length") or 0) | |
| if length <= 0: | |
| return {} | |
| raw = self.rfile.read(length) | |
| try: | |
| return json.loads(raw.decode("utf-8")) | |
| except json.JSONDecodeError: | |
| return {"_raw": raw.decode("utf-8", errors="replace")} | |
| def _json(self, status: int, payload: dict[str, Any]) -> None: | |
| raw = json.dumps(payload, sort_keys=True).encode("utf-8") | |
| self.send_response(status) | |
| self.send_header("content-type", "application/json") | |
| self.send_header("content-length", str(len(raw))) | |
| self.end_headers() | |
| self.wfile.write(raw) | |
| def _sse(self, entry: dict[str, Any], events: list[bytes], close_after_events: int | None = None) -> None: | |
| self.send_response(200) | |
| self.send_header("content-type", "text/event-stream") | |
| self.send_header("cache-control", "no-cache") | |
| self.end_headers() | |
| seen_stop = False | |
| sent = 0 | |
| for event in events: | |
| if event.startswith(b": delay-ms:"): | |
| try: | |
| time.sleep(int(event.decode().split(":", 2)[2]) / 1000.0) | |
| except (IndexError, ValueError): | |
| pass | |
| continue | |
| sent += 1 | |
| decoded = event.decode("utf-8", errors="replace") | |
| event_entry = { | |
| "raw": decoded[:500], | |
| "after_stop": seen_stop, | |
| "is_stop": is_stop_event(decoded), | |
| "broken_pipe": False, | |
| } | |
| entry["response_events"].append(event_entry) | |
| try: | |
| self.wfile.write(event) | |
| self.wfile.flush() | |
| except BrokenPipeError: | |
| entry["broken_pipe"] = True | |
| event_entry["broken_pipe"] = True | |
| return | |
| if event_entry["is_stop"]: | |
| seen_stop = True | |
| if close_after_events is not None and sent >= close_after_events: | |
| entry["closed_mid_stream"] = True | |
| self.close_connection = True | |
| return | |
| def select_response(case: dict[str, Any], body: dict[str, Any], requests_before: int) -> dict[str, Any]: | |
| if has_tool_result(body): | |
| sequence = case.get("provider_followup_sequence") | |
| if isinstance(sequence, list): | |
| index = max(0, requests_before - 1) | |
| if index < len(sequence) and isinstance(sequence[index], dict): | |
| followup = sequence[index] | |
| if followup.get("kind") == "error": | |
| return followup | |
| return { | |
| "kind": "stream", | |
| "steps": adapt_steps_to_tools(followup.get("steps", []), body, requests_before), | |
| "close_after_events": followup.get("close_after_events"), | |
| } | |
| followup = case.get("provider_followup") | |
| if isinstance(followup, dict): | |
| if followup.get("kind") == "error": | |
| return followup | |
| if isinstance(followup.get("steps"), list): | |
| return { | |
| "kind": "stream", | |
| "steps": adapt_steps_to_tools(followup["steps"], body, requests_before), | |
| "close_after_events": followup.get("close_after_events"), | |
| } | |
| retry = case.get("provider_retry") | |
| if isinstance(retry, dict): | |
| after_requests = int(retry.get("after_requests", 1)) | |
| if requests_before >= after_requests: | |
| if retry.get("kind") == "error": | |
| return retry | |
| if isinstance(retry.get("steps"), list): | |
| return { | |
| "kind": "stream", | |
| "steps": adapt_steps_to_tools(retry["steps"], body, requests_before), | |
| "close_after_events": retry.get("close_after_events"), | |
| } | |
| return { | |
| "kind": "stream", | |
| "steps": adapt_steps_to_tools(case.get("steps", []), body, requests_before), | |
| "close_after_events": case.get("close_after_events"), | |
| } | |
| def render_step_templates(value: Any, request_index: int) -> Any: | |
| if isinstance(value, str): | |
| return ( | |
| value.replace("{{request_index}}", str(request_index)) | |
| .replace("{{request_number}}", str(request_index + 1)) | |
| ) | |
| if isinstance(value, list): | |
| return [render_step_templates(item, request_index) for item in value] | |
| if isinstance(value, dict): | |
| return {key: render_step_templates(item, request_index) for key, item in value.items()} | |
| return value | |
| def adapt_steps_to_tools(steps: list[dict[str, Any]], body: dict[str, Any], request_index: int = 0) -> list[dict[str, Any]]: | |
| advertised = advertised_tool_schemas(body) | |
| if not advertised: | |
| return [render_step_templates(step, request_index) for step in steps] | |
| shell_tool = choose_shell_tool(advertised) | |
| if not shell_tool: | |
| return [render_step_templates(step, request_index) for step in steps] | |
| shell_name, shell_arg = shell_tool | |
| shell_call_ids: set[str] = set() | |
| out: list[dict[str, Any]] = [] | |
| changed = False | |
| for action, payload in normalized_steps(steps): | |
| payload = render_step_templates(dict(payload), request_index) | |
| if action == "provider.stream.tool_call_start": | |
| name = str(payload.get("name", "")) | |
| if name in {"Bash", "Shell", "bash", "shell"} and name not in advertised: | |
| payload["name"] = shell_name | |
| shell_call_ids.add(str(payload.get("id", ""))) | |
| changed = True | |
| elif name == "Edit" and "Edit" not in advertised and "edit" in advertised: | |
| payload["name"] = "edit" | |
| changed = True | |
| elif action == "provider.stream.tool_args_delta" and str(payload.get("id", "")) in shell_call_ids: | |
| adapted = adapt_shell_arguments(str(payload.get("delta", "")), shell_arg, advertised.get(shell_name, set())) | |
| if adapted != payload.get("delta"): | |
| payload["delta"] = adapted | |
| changed = True | |
| out.append({action: payload}) | |
| return out if changed else steps | |
| def advertised_tool_schemas(body: dict[str, Any]) -> dict[str, set[str]]: | |
| schemas: dict[str, set[str]] = {} | |
| raw_tools = body.get("tools") | |
| if not isinstance(raw_tools, list): | |
| return schemas | |
| for item in raw_tools: | |
| if not isinstance(item, dict): | |
| continue | |
| function = item.get("function") | |
| if isinstance(function, dict): | |
| name = str(function.get("name", "")) | |
| params = function.get("parameters", {}) | |
| else: | |
| name = str(item.get("name", "")) | |
| params = item.get("parameters") or item.get("input_schema") or {} | |
| properties = params.get("properties", {}) if isinstance(params, dict) else {} | |
| if name: | |
| schemas[name] = set(str(key) for key in properties) if isinstance(properties, dict) else set() | |
| return schemas | |
| def choose_shell_tool(advertised: dict[str, set[str]]) -> tuple[str, str] | None: | |
| candidates = ( | |
| ("exec_command", "cmd"), | |
| ("run_shell_command", "command"), | |
| ("run_commands", "commands"), | |
| ("terminal", "command"), | |
| ("run_command", "command"), | |
| ("shell", "command"), | |
| ("bash", "command"), | |
| ("Bash", "command"), | |
| ) | |
| for name, preferred_arg in candidates: | |
| if name not in advertised: | |
| continue | |
| properties = advertised[name] | |
| if preferred_arg in properties or not properties: | |
| return name, preferred_arg | |
| if "command" in properties: | |
| return name, "command" | |
| if "cmd" in properties: | |
| return name, "cmd" | |
| if "commands" in properties: | |
| return name, "commands" | |
| return None | |
| def adapt_shell_arguments(raw: str, arg_name: str, properties: set[str]) -> str: | |
| try: | |
| args = json.loads(raw) | |
| except json.JSONDecodeError: | |
| return raw | |
| if not isinstance(args, dict): | |
| return raw | |
| command = args.get("command") or args.get("cmd") | |
| if not isinstance(command, str): | |
| return raw | |
| adapted = dict(args) | |
| adapted.pop("command", None) | |
| adapted.pop("cmd", None) | |
| adapted[arg_name] = [command] if arg_name == "commands" else command | |
| if "description" in properties and "description" not in adapted: | |
| adapted["description"] = "standalone shell repro" | |
| return json.dumps(adapted, separators=(",", ":")) | |
| def wire_for_path(path: str) -> str: | |
| if path.startswith("/v1/messages"): | |
| return "anthropic" | |
| if path == "/v1/responses": | |
| return "openai_responses" | |
| if path == "/v1/chat/completions": | |
| return "openai_chat" | |
| return "unknown" | |
| def is_stop_event(raw: str) -> bool: | |
| return "message_stop" in raw or "response.completed" in raw or raw.strip() == "data: [DONE]" | |
| def encode_stream(case: dict[str, Any], steps: list[dict[str, Any]], wire: str, model: str) -> list[bytes]: | |
| if wire == "anthropic": | |
| return encode_anthropic(steps, model or "claude-3-5-sonnet-20241022", bool(case.get("anthropic_full_input"))) | |
| if wire == "openai_responses": | |
| return encode_openai_responses(steps, model or "gpt-4") | |
| if wire == "openai_chat": | |
| return encode_openai_chat(steps, model or "gpt-4") | |
| return [sse("error", {"error": {"message": "unsupported path"}})] | |
| def normalized_steps(steps: list[dict[str, Any]]) -> list[tuple[str, dict[str, Any]]]: | |
| out = [] | |
| for item in steps: | |
| if not isinstance(item, dict) or len(item) != 1: | |
| raise ValueError(f"invalid step: {item!r}") | |
| action, payload = next(iter(item.items())) | |
| out.append((str(action), payload if isinstance(payload, dict) else {})) | |
| return out | |
| def tool_args_by_id(steps: list[dict[str, Any]]) -> dict[str, str]: | |
| out: dict[str, str] = {} | |
| for action, payload in normalized_steps(steps): | |
| if action == "provider.stream.tool_args_delta": | |
| call_id = str(payload.get("id", "")) | |
| out[call_id] = out.get(call_id, "") + str(payload.get("delta", "")) | |
| return out | |
| def encode_anthropic(steps: list[dict[str, Any]], model: str, full_input: bool = False) -> list[bytes]: | |
| events: list[bytes] = [] | |
| args_by_id = tool_args_by_id(steps) | |
| message_started = False | |
| next_index = 0 | |
| open_index: int | None = None | |
| index_by_id: dict[str, int] = {} | |
| def message_start() -> None: | |
| nonlocal message_started | |
| if message_started: | |
| return | |
| message_started = True | |
| events.append( | |
| sse( | |
| "message_start", | |
| { | |
| "type": "message_start", | |
| "message": { | |
| "id": "msg_repro", | |
| "type": "message", | |
| "role": "assistant", | |
| "model": model, | |
| "content": [], | |
| "stop_reason": None, | |
| "usage": {"input_tokens": 1, "output_tokens": 1}, | |
| }, | |
| }, | |
| ) | |
| ) | |
| def block_stop() -> None: | |
| nonlocal open_index | |
| if open_index is not None: | |
| events.append(sse("content_block_stop", {"type": "content_block_stop", "index": open_index})) | |
| open_index = None | |
| for action, payload in normalized_steps(steps): | |
| if action == "provider.stream.message_start": | |
| message_start() | |
| elif action == "provider.stream.tool_call_start": | |
| message_start() | |
| block_stop() | |
| call_id = str(payload["id"]) | |
| name = str(payload["name"]) | |
| open_index = next_index | |
| index_by_id[call_id] = open_index | |
| next_index += 1 | |
| events.append( | |
| sse( | |
| "content_block_start", | |
| { | |
| "type": "content_block_start", | |
| "index": open_index, | |
| "content_block": { | |
| "type": "tool_use", | |
| "id": call_id, | |
| "name": name, | |
| "input": parse_json_object(args_by_id.get(call_id, "")) if full_input else {}, | |
| }, | |
| }, | |
| ) | |
| ) | |
| elif action == "provider.stream.tool_args_delta": | |
| call_id = str(payload.get("id", "")) | |
| if full_input and call_id in index_by_id: | |
| continue | |
| message_start() | |
| events.append( | |
| sse( | |
| "content_block_delta", | |
| { | |
| "type": "content_block_delta", | |
| "index": index_by_id.get(call_id, open_index or 0), | |
| "delta": {"type": "input_json_delta", "partial_json": str(payload.get("delta", ""))}, | |
| }, | |
| ) | |
| ) | |
| elif action == "provider.stream.tool_call_stop": | |
| block_stop() | |
| elif action == "provider.stream.message_stop": | |
| block_stop() | |
| events.append(sse("message_stop", {"type": "message_stop"})) | |
| elif action == "provider.time.delay_ms": | |
| events.append(f": delay-ms:{int(payload.get('value', 0))}\n\n".encode()) | |
| if message_started and not any(b"message_stop" in event for event in events): | |
| block_stop() | |
| events.append(sse("message_stop", {"type": "message_stop"})) | |
| return events | |
| def encode_openai_responses(steps: list[dict[str, Any]], model: str) -> list[bytes]: | |
| response_id = "resp_repro" | |
| events = [ | |
| sse( | |
| "response.created", | |
| { | |
| "type": "response.created", | |
| "response": { | |
| "id": response_id, | |
| "object": "response", | |
| "created_at": int(time.time()), | |
| "status": "in_progress", | |
| "model": model, | |
| "output": [], | |
| }, | |
| }, | |
| ), | |
| sse("response.in_progress", {"type": "response.in_progress", "response": {"id": response_id, "status": "in_progress"}}), | |
| ] | |
| output_index = 0 | |
| item_by_id: dict[str, dict[str, Any]] = {} | |
| args_by_id: dict[str, str] = {} | |
| for action, payload in normalized_steps(steps): | |
| if action == "provider.stream.tool_call_start": | |
| call_id = str(payload["id"]) | |
| item = { | |
| "id": "fc_" + call_id, | |
| "type": "function_call", | |
| "call_id": call_id, | |
| "name": str(payload["name"]), | |
| "arguments": "", | |
| "status": "in_progress", | |
| } | |
| item_by_id[call_id] = item | |
| args_by_id.setdefault(call_id, "") | |
| events.append( | |
| sse( | |
| "response.output_item.added", | |
| { | |
| "type": "response.output_item.added", | |
| "output_index": output_index, | |
| "item": item, | |
| }, | |
| ) | |
| ) | |
| elif action == "provider.stream.tool_args_delta": | |
| call_id = str(payload.get("id", "")) | |
| delta = str(payload.get("delta", "")) | |
| args_by_id[call_id] = args_by_id.get(call_id, "") + delta | |
| events.append( | |
| sse( | |
| "response.function_call_arguments.delta", | |
| { | |
| "type": "response.function_call_arguments.delta", | |
| "output_index": output_index, | |
| "delta": delta, | |
| }, | |
| ) | |
| ) | |
| elif action == "provider.stream.tool_call_stop": | |
| call_id = str(payload.get("id", "")) | |
| item = dict(item_by_id.get(call_id, {})) | |
| if item: | |
| item["status"] = "completed" | |
| item["arguments"] = args_by_id.get(call_id, "") | |
| item_by_id[call_id] = item | |
| events.append( | |
| sse( | |
| "response.function_call_arguments.done", | |
| { | |
| "type": "response.function_call_arguments.done", | |
| "output_index": output_index, | |
| "arguments": args_by_id.get(call_id, ""), | |
| }, | |
| ) | |
| ) | |
| events.append( | |
| sse( | |
| "response.output_item.done", | |
| { | |
| "type": "response.output_item.done", | |
| "output_index": output_index, | |
| "item": item, | |
| }, | |
| ) | |
| ) | |
| output_index += 1 | |
| elif action == "provider.stream.message_stop": | |
| events.append( | |
| sse( | |
| "response.completed", | |
| { | |
| "type": "response.completed", | |
| "response": { | |
| "id": response_id, | |
| "object": "response", | |
| "status": "completed", | |
| "model": model, | |
| "output": list(item_by_id.values()), | |
| "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}, | |
| }, | |
| }, | |
| ) | |
| ) | |
| elif action == "provider.time.delay_ms": | |
| events.append(f": delay-ms:{int(payload.get('value', 0))}\n\n".encode()) | |
| if not any(b"response.completed" in event for event in events): | |
| events.append( | |
| sse( | |
| "response.completed", | |
| { | |
| "type": "response.completed", | |
| "response": { | |
| "id": response_id, | |
| "object": "response", | |
| "status": "completed", | |
| "model": model, | |
| "output": list(item_by_id.values()), | |
| "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}, | |
| }, | |
| }, | |
| ) | |
| ) | |
| return events | |
| def encode_openai_chat(steps: list[dict[str, Any]], model: str) -> list[bytes]: | |
| events: list[bytes] = [] | |
| chat_id = "chatcmpl_repro" | |
| started = False | |
| next_index = 0 | |
| index_by_id: dict[str, int] = {} | |
| def start() -> None: | |
| nonlocal started | |
| if started: | |
| return | |
| started = True | |
| events.append( | |
| sse( | |
| "", | |
| { | |
| "id": chat_id, | |
| "object": "chat.completion.chunk", | |
| "created": int(time.time()), | |
| "model": model, | |
| "choices": [{"index": 0, "delta": {"role": "assistant"}, "finish_reason": None}], | |
| }, | |
| ) | |
| ) | |
| has_tool_calls = False | |
| for action, payload in normalized_steps(steps): | |
| if action == "provider.stream.tool_call_start": | |
| start() | |
| has_tool_calls = True | |
| call_id = str(payload["id"]) | |
| index_by_id[call_id] = next_index | |
| next_index += 1 | |
| events.append( | |
| sse( | |
| "", | |
| { | |
| "choices": [ | |
| { | |
| "index": 0, | |
| "delta": { | |
| "tool_calls": [ | |
| { | |
| "index": index_by_id[call_id], | |
| "id": call_id, | |
| "type": "function", | |
| "function": {"name": str(payload["name"]), "arguments": ""}, | |
| } | |
| ] | |
| }, | |
| "finish_reason": None, | |
| } | |
| ] | |
| }, | |
| ) | |
| ) | |
| elif action == "provider.stream.tool_args_delta": | |
| start() | |
| call_id = str(payload.get("id", "")) | |
| events.append( | |
| sse( | |
| "", | |
| { | |
| "choices": [ | |
| { | |
| "index": 0, | |
| "delta": { | |
| "tool_calls": [ | |
| { | |
| "index": index_by_id.get(call_id, 0), | |
| "function": {"arguments": str(payload.get("delta", ""))}, | |
| } | |
| ] | |
| }, | |
| "finish_reason": None, | |
| } | |
| ] | |
| }, | |
| ) | |
| ) | |
| elif action == "provider.stream.message_stop": | |
| start() | |
| events.append( | |
| sse( | |
| "", | |
| { | |
| "id": chat_id, | |
| "object": "chat.completion.chunk", | |
| "created": int(time.time()), | |
| "model": model, | |
| "choices": [ | |
| { | |
| "index": 0, | |
| "delta": {}, | |
| "finish_reason": "tool_calls" if has_tool_calls else "stop", | |
| } | |
| ], | |
| }, | |
| ) | |
| ) | |
| events.append(b"data: [DONE]\n\n") | |
| elif action == "provider.time.delay_ms": | |
| events.append(f": delay-ms:{int(payload.get('value', 0))}\n\n".encode()) | |
| if not events or events[-1] != b"data: [DONE]\n\n": | |
| start() | |
| events.append( | |
| sse( | |
| "", | |
| { | |
| "id": chat_id, | |
| "object": "chat.completion.chunk", | |
| "created": int(time.time()), | |
| "model": model, | |
| "choices": [ | |
| { | |
| "index": 0, | |
| "delta": {}, | |
| "finish_reason": "tool_calls" if has_tool_calls else "stop", | |
| } | |
| ], | |
| }, | |
| ) | |
| ) | |
| events.append(b"data: [DONE]\n\n") | |
| return events | |
| def parse_json_object(value: str) -> dict[str, Any]: | |
| try: | |
| parsed = json.loads(value or "{}") | |
| except json.JSONDecodeError: | |
| return {} | |
| return parsed if isinstance(parsed, dict) else {} | |
| def sse(event: str, payload: dict[str, Any]) -> bytes: | |
| prefix = f"event: {event}\n" if event else "" | |
| return (prefix + "data: " + json.dumps(payload, sort_keys=True) + "\n\n").encode() | |
| def has_tool_result(value: Any) -> bool: | |
| if isinstance(value, list): | |
| return any(has_tool_result(item) for item in value) | |
| if not isinstance(value, dict): | |
| return False | |
| raw_type = str(value.get("type", "")).lower() | |
| role = str(value.get("role", "")).lower() | |
| if raw_type in {"function_call_output", "tool_result", "tool_use_result", "tool_call_output"}: | |
| return True | |
| if role == "tool": | |
| return True | |
| return any(has_tool_result(item) for item in value.values()) | |
| def collect_tool_results(value: Any, out: list[dict[str, Any]]) -> None: | |
| if isinstance(value, list): | |
| for item in value: | |
| collect_tool_results(item, out) | |
| return | |
| if not isinstance(value, dict): | |
| return | |
| raw_type = str(value.get("type", "")).lower() | |
| role = str(value.get("role", "")).lower() | |
| result_like = raw_type in {"function_call_output", "tool_result", "tool_use_result", "tool_call_output"} or role == "tool" | |
| if result_like: | |
| call_id = "" | |
| for key in ("call_id", "tool_call_id", "tool_use_id"): | |
| if value.get(key): | |
| call_id = str(value[key]) | |
| break | |
| if call_id: | |
| payload = value.get("output") if "output" in value else value.get("result") if "result" in value else value.get("content") | |
| out.append( | |
| { | |
| "call_id": call_id, | |
| "type": raw_type or role or "tool_result", | |
| "payload_text": json.dumps(payload, default=str, sort_keys=True), | |
| } | |
| ) | |
| for item in value.values(): | |
| collect_tool_results(item, out) | |
| def build_image(case: dict[str, Any], image: str) -> None: | |
| dockerfile = str( | |
| case.get("dockerfile") | |
| or dockerfile_for_target(str(case["target"]), str(case.get("version", image.rsplit(":", 1)[-1]))) | |
| ) | |
| with tempfile.TemporaryDirectory(prefix="vendor-repro-build-") as tmp: | |
| path = Path(tmp) / "Dockerfile" | |
| path.write_text(dockerfile, encoding="utf-8") | |
| cmd = [ | |
| "docker", | |
| "build", | |
| "--build-arg", | |
| f"RUNNER_UID={os.getuid()}", | |
| "--build-arg", | |
| f"RUNNER_GID={os.getgid()}", | |
| "-t", | |
| image, | |
| "-f", | |
| str(path), | |
| tmp, | |
| ] | |
| run_checked(cmd) | |
| def run_checked(cmd: list[str]) -> None: | |
| print("+ " + " ".join(cmd), flush=True) | |
| proc = subprocess.run(cmd, text=True) | |
| if proc.returncode != 0: | |
| raise SystemExit(proc.returncode) | |
| def derive_image(case: dict[str, Any]) -> str: | |
| if case.get("image"): | |
| return str(case["image"]) | |
| return f"vendor-repro-{case['target']}:{case.get('version', 'latest')}" | |
| def target_command(target: str, base_url: str, workspace: str, timeout_s: int) -> tuple[list[str], dict[str, str]]: | |
| if target == "claude": | |
| return ( | |
| [ | |
| "claude", | |
| "-p", | |
| PROMPT, | |
| "--dangerously-skip-permissions", | |
| "--output-format", | |
| "text", | |
| "--model", | |
| "claude-3-5-sonnet-20241022", | |
| ], | |
| { | |
| "ANTHROPIC_BASE_URL": base_url, | |
| "ANTHROPIC_API_KEY": "vendor-repro-fake-key", | |
| "ANTHROPIC_AUTH_TOKEN": "vendor-repro-fake-key", | |
| "DISABLE_AUTOUPDATER": "1", | |
| "CLAUDE_CODE_DISABLE_TELEMETRY": "1", | |
| "DISABLE_TELEMETRY": "1", | |
| "DISABLE_ERROR_REPORTING": "1", | |
| "DISABLE_NON_ESSENTIAL_MODEL_CALLS": "1", | |
| }, | |
| ) | |
| if target == "codex": | |
| base = base_url.rstrip("/") | |
| return ( | |
| [ | |
| "codex", | |
| "exec", | |
| "-c", | |
| 'model_provider="vendor-repro"', | |
| "-c", | |
| 'model_providers.vendor-repro.name="vendor-repro"', | |
| "-c", | |
| f'model_providers.vendor-repro.base_url="{base}/v1"', | |
| "-c", | |
| 'model_providers.vendor-repro.wire_api="responses"', | |
| "-c", | |
| "model_providers.vendor-repro.requires_openai_auth=false", | |
| "-c", | |
| 'approval_policy="never"', | |
| "-c", | |
| 'sandbox_mode="danger-full-access"', | |
| "-c", | |
| 'model="gpt-4"', | |
| "-c", | |
| 'model_reasoning_effort="low"', | |
| "--skip-git-repo-check", | |
| PROMPT, | |
| ], | |
| {"OPENAI_API_KEY": "vendor-repro-fake-key"}, | |
| ) | |
| if target == "goose": | |
| return ( | |
| [ | |
| "goose", | |
| "run", | |
| "--no-profile", | |
| "--with-builtin", | |
| "developer", | |
| "--max-turns", | |
| str(max(3, timeout_s // 10)), | |
| "--output-format", | |
| "json", | |
| "--text", | |
| PROMPT, | |
| ], | |
| { | |
| "ANTHROPIC_API_KEY": "vendor-repro-fake-key", | |
| "ANTHROPIC_HOST": base_url.rstrip(), | |
| "GOOSE_PROVIDER": "anthropic", | |
| "GOOSE_MODEL": "claude-sonnet-4-5", | |
| }, | |
| ) | |
| if target == "crush": | |
| return ( | |
| [ | |
| "crush", | |
| "--cwd", | |
| workspace, | |
| "--data-dir", | |
| workspace + "/vendor_repro_crush_home/.local/share/crush", | |
| "run", | |
| "--quiet", | |
| "-m", | |
| "local-repro/gpt-4", | |
| PROMPT, | |
| ], | |
| {"OPENAI_API_KEY": "vendor-repro-fake-key"}, | |
| ) | |
| if target == "qwen": | |
| return ( | |
| [ | |
| "/home/runner/.npm-global/bin/qwen", | |
| "--chat-recording=false", | |
| "--auth-type", | |
| "openai", | |
| "--openai-api-key", | |
| "vendor-repro-fake-key", | |
| "--openai-base-url", | |
| base_url.rstrip("/") + "/v1", | |
| "--model", | |
| "gpt-4", | |
| "--approval-mode", | |
| "yolo", | |
| "--output-format", | |
| "text", | |
| "-p", | |
| PROMPT, | |
| ], | |
| {"OPENAI_API_KEY": "vendor-repro-fake-key", "QWEN_CODE_SUPPRESS_YOLO_WARNING": "1"}, | |
| ) | |
| if target == "cline": | |
| base = base_url.rstrip("/") + "/v1" | |
| data_dir = "/workspace/vendor_repro_cline_data" | |
| model = "gpt-4" | |
| auth_cmd = " ".join( | |
| [ | |
| shlex.quote("cline"), | |
| "auth", | |
| "--provider", | |
| "openai-native", | |
| "--apikey", | |
| "vendor-repro-fake-key", | |
| "--modelid", | |
| shlex.quote(model), | |
| "--baseurl", | |
| shlex.quote(base), | |
| "--data-dir", | |
| shlex.quote(data_dir), | |
| ] | |
| ) | |
| run_cmd = " ".join( | |
| [ | |
| "exec", | |
| shlex.quote("cline"), | |
| "--data-dir", | |
| shlex.quote(data_dir), | |
| "--cwd", | |
| shlex.quote(workspace), | |
| "-P", | |
| "openai-native", | |
| "-m", | |
| shlex.quote(model), | |
| "-k", | |
| "vendor-repro-fake-key", | |
| "--json", | |
| "--retries", | |
| "100", | |
| "-y", | |
| shlex.quote(PROMPT), | |
| ] | |
| ) | |
| return ( | |
| ["bash", "-c", auth_cmd + " && " + run_cmd], | |
| { | |
| "CLINE_DATA_DIR": data_dir, | |
| "CLINE_LOG_ENABLED": "0", | |
| "OPENAI_API_KEY": "vendor-repro-fake-key", | |
| }, | |
| ) | |
| if target == "opencode": | |
| return ( | |
| ["opencode", "run", "-m", "anthropic/claude-haiku-4-5-20251001", PROMPT], | |
| { | |
| "ANTHROPIC_API_KEY": "vendor-repro-fake-key", | |
| "ANTHROPIC_BASE_URL": base_url.rstrip("/") + "/v1", | |
| "OPENCODE_DISABLE_DEFAULT_PLUGINS": "1", | |
| }, | |
| ) | |
| if target == "continue": | |
| return ( | |
| [ | |
| "cn", | |
| "--config", | |
| "/workspace/vendor_repro_continue_home/config.yaml", | |
| "--auto", | |
| "--silent", | |
| "-p", | |
| PROMPT, | |
| ], | |
| { | |
| "CONTINUE_HOME": "/workspace/vendor_repro_continue_home", | |
| "OPENAI_API_KEY": "vendor-repro-fake-key", | |
| }, | |
| ) | |
| raise ValueError(f"unsupported target: {target}") | |
| def prepare_workspace(root: Path, target: str, base_url: str, case: dict[str, Any]) -> dict[str, str]: | |
| root.mkdir(parents=True, exist_ok=True) | |
| (root / "README.md").write_text("# Vendor repro workspace\n", encoding="utf-8") | |
| files = case.get("workspace_files", {}) | |
| if isinstance(files, dict): | |
| for raw_path, content in files.items(): | |
| rel = Path(str(raw_path)) | |
| if rel.is_absolute() or ".." in rel.parts: | |
| raise ValueError(f"unsafe workspace path: {raw_path}") | |
| destination = root / rel | |
| destination.parent.mkdir(parents=True, exist_ok=True) | |
| destination.write_text(str(content), encoding="utf-8") | |
| env: dict[str, str] = {} | |
| if target == "crush": | |
| host_home = root / "vendor_repro_crush_home" | |
| host_config_dir = host_home / ".config" / "crush" | |
| host_data_dir = host_home / ".local" / "share" / "crush" | |
| host_config_dir.mkdir(parents=True, exist_ok=True) | |
| host_data_dir.mkdir(parents=True, exist_ok=True) | |
| (host_config_dir / "crush.json").write_text( | |
| json.dumps( | |
| { | |
| "$schema": "https://charm.land/crush.json", | |
| "providers": { | |
| "local-repro": { | |
| "type": "openai-compat", | |
| "base_url": base_url.rstrip("/") + "/v1", | |
| "api_key": "vendor-repro-fake-key", | |
| "models": [ | |
| { | |
| "id": "gpt-4", | |
| "name": "Local repro gpt-4", | |
| "context_window": 128000, | |
| "default_max_tokens": 2000, | |
| } | |
| ], | |
| } | |
| }, | |
| }, | |
| sort_keys=True, | |
| ), | |
| encoding="utf-8", | |
| ) | |
| env.update( | |
| { | |
| "HOME": "/workspace/vendor_repro_crush_home", | |
| "XDG_CONFIG_HOME": "/workspace/vendor_repro_crush_home/.config", | |
| "XDG_DATA_HOME": "/workspace/vendor_repro_crush_home/.local/share", | |
| "CRUSH_GLOBAL_CONFIG": "/workspace/vendor_repro_crush_home/.config/crush", | |
| "CRUSH_GLOBAL_DATA": "/workspace/vendor_repro_crush_home/.local/share/crush", | |
| } | |
| ) | |
| if target == "goose": | |
| home = root / "vendor_repro_goose_home" | |
| for rel in (".config", ".cache", ".local/share", ".local/state"): | |
| (home / rel).mkdir(parents=True, exist_ok=True) | |
| env.update( | |
| { | |
| "HOME": "/workspace/vendor_repro_goose_home", | |
| "XDG_CONFIG_HOME": "/workspace/vendor_repro_goose_home/.config", | |
| "XDG_CACHE_HOME": "/workspace/vendor_repro_goose_home/.cache", | |
| "XDG_DATA_HOME": "/workspace/vendor_repro_goose_home/.local/share", | |
| "XDG_STATE_HOME": "/workspace/vendor_repro_goose_home/.local/state", | |
| } | |
| ) | |
| if target == "qwen": | |
| host_home = root / "vendor_repro_qwen_home" | |
| host_config = host_home / ".qwen" | |
| host_config.mkdir(parents=True, exist_ok=True) | |
| (host_config / "settings.json").write_text( | |
| json.dumps( | |
| { | |
| "modelProviders": { | |
| "openai": [ | |
| { | |
| "id": "gpt-4", | |
| "name": "gpt-4", | |
| "baseUrl": base_url.rstrip("/") + "/v1", | |
| "envKey": "OPENAI_API_KEY", | |
| } | |
| ] | |
| }, | |
| "env": {"OPENAI_API_KEY": "vendor-repro-fake-key"}, | |
| "security": {"auth": {"selectedType": "openai"}}, | |
| "model": {"name": "gpt-4"}, | |
| "telemetry": {"enabled": False}, | |
| }, | |
| sort_keys=True, | |
| ), | |
| encoding="utf-8", | |
| ) | |
| env["HOME"] = "/workspace/vendor_repro_qwen_home" | |
| if target == "cline": | |
| host_data = root / "vendor_repro_cline_data" | |
| host_data.mkdir(parents=True, exist_ok=True) | |
| host_data.chmod(0o777) | |
| env["CLINE_DATA_DIR"] = "/workspace/vendor_repro_cline_data" | |
| if target == "continue": | |
| host_home = root / "vendor_repro_continue_home" | |
| host_home.mkdir(parents=True, exist_ok=True) | |
| config_path = host_home / "config.yaml" | |
| config_path.write_text( | |
| "name: Vendor Reproducer\n" | |
| "version: 1.0.0\n" | |
| "schema: v1\n" | |
| "models:\n" | |
| " - name: Vendor GPT-4\n" | |
| " provider: openai\n" | |
| " model: gpt-4\n" | |
| " apiKey: vendor-repro-fake-key\n" | |
| " apiBase: " + base_url.rstrip("/") + "/v1\n", | |
| encoding="utf-8", | |
| ) | |
| config_path.chmod(0o666) | |
| env["CONTINUE_HOME"] = "/workspace/vendor_repro_continue_home" | |
| if target == "opencode": | |
| host_home = root / "vendor_repro_opencode_home" | |
| for rel in (".config", ".cache", ".local/share", ".local/share/opencode"): | |
| (host_home / rel).mkdir(parents=True, exist_ok=True) | |
| env.update( | |
| { | |
| "HOME": "/workspace/vendor_repro_opencode_home", | |
| "XDG_CONFIG_HOME": "/workspace/vendor_repro_opencode_home/.config", | |
| "XDG_CACHE_HOME": "/workspace/vendor_repro_opencode_home/.cache", | |
| "XDG_DATA_HOME": "/workspace/vendor_repro_opencode_home/.local/share", | |
| "OPENCODE_DISABLE_DEFAULT_PLUGINS": "1", | |
| } | |
| ) | |
| if case.get("side_effect_probe"): | |
| env["REPRO_SIDE_EFFECT_ROOT"] = "/side-effects" | |
| return env | |
| def run_target( | |
| case: dict[str, Any], | |
| image: str, | |
| base_url: str, | |
| workspace: Path, | |
| side_effects: Path, | |
| extra_env: dict[str, str], | |
| ) -> subprocess.CompletedProcess[bytes]: | |
| target = str(case["target"]) | |
| timeout_s = int(case.get("timeout_s", 90)) | |
| inner, env = target_command(target, base_url, "/workspace", timeout_s) | |
| env = {**extra_env, **env} | |
| cmd = [ | |
| "docker", | |
| "run", | |
| "--rm", | |
| "-v", | |
| f"{workspace}:/workspace:rw", | |
| "-v", | |
| f"{side_effects}:/side-effects:rw", | |
| "-w", | |
| "/workspace", | |
| *docker_network_args(base_url), | |
| "--cpus", | |
| str(case.get("docker_cpus", DEFAULT_DOCKER_CPUS)), | |
| "--memory", | |
| str(case.get("docker_memory", DEFAULT_DOCKER_MEMORY)), | |
| "-e", | |
| f"PATH={DOCKER_PATH}", | |
| ] | |
| for key, value in sorted(env.items()): | |
| cmd.extend(["-e", f"{key}={value}"]) | |
| docker_user = os.environ.get("MODELLOCK_REPRO_DOCKER_USER") | |
| if docker_user is None and target == "claude": | |
| docker_user = f"{os.getuid()}:{os.getgid()}" | |
| cmd.extend(["--entrypoint", "/usr/bin/env"]) | |
| if docker_user: | |
| cmd.extend(["--user", docker_user]) | |
| cmd.append(image) | |
| if case.get("side_effect_probe"): | |
| pid_file = str(case.get("pid_file", "child.pid")) | |
| alive_file = str(case.get("alive_file", "child.alive")) | |
| quoted_inner = " ".join(shlex.quote(part) for part in inner) | |
| probe = ( | |
| "status=0; " | |
| + quoted_inner | |
| + " || status=$?; " | |
| + "python3 - <<'PY'\n" | |
| + "import os, pathlib\n" | |
| + "root = pathlib.Path('/side-effects')\n" | |
| + f"pid_path = root / {pid_file!r}\n" | |
| + f"alive_path = root / {alive_file!r}\n" | |
| + "alive = False\n" | |
| + "pid_text = ''\n" | |
| + "if pid_path.exists():\n" | |
| + " pid_text = pid_path.read_text().strip()\n" | |
| + " try:\n" | |
| + " os.kill(int(pid_text), 0)\n" | |
| + " alive = True\n" | |
| + " except Exception:\n" | |
| + " alive = False\n" | |
| + "alive_path.write_text(('alive' if alive else 'dead') + '\\n' + pid_text)\n" | |
| + "PY\n" | |
| + "exit $status" | |
| ) | |
| cmd.extend(["bash", "-c", probe]) | |
| else: | |
| cmd.extend(inner) | |
| print("+ " + " ".join(cmd), flush=True) | |
| proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, start_new_session=True) | |
| signal_after = case.get("signal_after_ms") | |
| if signal_after is not None: | |
| def send_signal() -> None: | |
| time.sleep(float(signal_after) / 1000.0) | |
| try: | |
| os.killpg(os.getpgid(proc.pid), signal.SIGINT) | |
| except ProcessLookupError: | |
| pass | |
| threading.Thread(target=send_signal, daemon=True).start() | |
| try: | |
| stdout, stderr = proc.communicate(timeout=timeout_s) | |
| result = subprocess.CompletedProcess(cmd, proc.returncode, stdout, stderr) | |
| except subprocess.TimeoutExpired: | |
| try: | |
| os.killpg(os.getpgid(proc.pid), signal.SIGTERM) | |
| except ProcessLookupError: | |
| pass | |
| try: | |
| stdout, stderr = proc.communicate(timeout=5) | |
| except subprocess.TimeoutExpired: | |
| try: | |
| os.killpg(os.getpgid(proc.pid), signal.SIGKILL) | |
| except ProcessLookupError: | |
| pass | |
| stdout, stderr = proc.communicate() | |
| result = subprocess.CompletedProcess(cmd, proc.returncode, stdout, stderr) | |
| result.timed_out = True # type: ignore[attr-defined] | |
| return result | |
| result.timed_out = False # type: ignore[attr-defined] | |
| return result | |
| def provider_host_for_container() -> str: | |
| override = os.environ.get("MODELLOCK_REPRO_PROVIDER_HOST") | |
| if override: | |
| return override | |
| if sys.platform.startswith("linux"): | |
| return "127.0.0.1" | |
| return "host.docker.internal" | |
| def docker_network_args(base_url: str) -> list[str]: | |
| if "127.0.0.1" in base_url or "localhost" in base_url: | |
| return ["--network", "host"] | |
| if "host.docker.internal" in base_url: | |
| return ["--add-host", "host.docker.internal:host-gateway"] | |
| return [] | |
| def temporary_workspace() -> Any: | |
| base = Path(os.environ.get("MODELLOCK_REPRO_TMPDIR", Path.cwd())).resolve() | |
| base.mkdir(parents=True, exist_ok=True) | |
| return _BestEffortTemporaryDirectory(prefix="vendor-repro-workspace-", dir=str(base)) | |
| class _BestEffortTemporaryDirectory: | |
| def __init__(self, prefix: str, dir: str) -> None: | |
| self.name = tempfile.mkdtemp(prefix=prefix, dir=dir) | |
| def __enter__(self) -> str: | |
| return self.name | |
| def __exit__(self, exc_type: object, exc: object, tb: object) -> None: | |
| try: | |
| shutil.rmtree(self.name) | |
| except Exception: | |
| pass | |
| def expanded_case_runs(case: dict[str, Any]) -> list[dict[str, Any]]: | |
| runs = case.get("runs") | |
| if not isinstance(runs, list): | |
| return [case] | |
| base = {key: value for key, value in case.items() if key != "runs"} | |
| expanded: list[dict[str, Any]] = [] | |
| for index, run in enumerate(runs, 1): | |
| if not isinstance(run, dict): | |
| raise ValueError(f"invalid run entry: {run!r}") | |
| merged = {**base, **run} | |
| merged.setdefault("run_name", f"run{index}") | |
| expanded.append(merged) | |
| return expanded | |
| def is_pid_alive(pid: int) -> bool: | |
| if pid <= 0: | |
| return False | |
| try: | |
| os.kill(pid, 0) | |
| except ProcessLookupError: | |
| return False | |
| except PermissionError: | |
| return True | |
| return True | |
| def response_events(requests: list[dict[str, Any]]) -> list[dict[str, Any]]: | |
| events: list[dict[str, Any]] = [] | |
| for request in requests: | |
| for event in request.get("response_events", []): | |
| if isinstance(event, dict): | |
| events.append(event) | |
| return events | |
| def max_tool_result_count_in_one_request(requests: list[dict[str, Any]], call_id: str = "") -> int: | |
| max_count = 0 | |
| for request in requests: | |
| results: list[dict[str, Any]] = [] | |
| collect_tool_results(request.get("body"), results) | |
| counts: dict[str, int] = {} | |
| for result in results: | |
| current = str(result.get("call_id", "")) | |
| if call_id and current != call_id: | |
| continue | |
| counts[current] = counts.get(current, 0) + 1 | |
| if counts: | |
| max_count = max(max_count, max(counts.values())) | |
| return max_count | |
| def evaluate( | |
| case: dict[str, Any], | |
| requests: list[dict[str, Any]], | |
| proc: subprocess.CompletedProcess[bytes], | |
| side_effects: Path, | |
| ) -> tuple[bool, list[str]]: | |
| results: list[dict[str, Any]] = [] | |
| for request in requests: | |
| collect_tool_results(request.get("body"), results) | |
| result_ids = [str(result["call_id"]) for result in results] | |
| counts: dict[str, int] = {} | |
| for call_id in result_ids: | |
| counts[call_id] = counts.get(call_id, 0) + 1 | |
| events = response_events(requests) | |
| late_events = [ | |
| event | |
| for event in events | |
| if event.get("after_stop") and not event.get("broken_pipe") and ("delta" in str(event.get("raw", ""))) | |
| ] | |
| total_result_bytes = sum(len(str(result.get("payload_text", "")).encode("utf-8", errors="ignore")) for result in results) | |
| closed_mid_stream = sum(1 for request in requests if request.get("closed_mid_stream")) | |
| timed_out = bool(getattr(proc, "timed_out", False)) | |
| checks = case.get("checks", []) | |
| messages = [ | |
| f"process_exit={proc.returncode}", | |
| f"process_timed_out={timed_out}", | |
| f"provider_requests={len(requests)}", | |
| f"tool_result_counts={dict(sorted(counts.items()))}", | |
| f"tool_result_bytes={total_result_bytes}", | |
| f"closed_mid_stream={closed_mid_stream}", | |
| f"late_stream_events_after_stop={len(late_events)}", | |
| ] | |
| ok = True | |
| for check in checks: | |
| kind = check.get("kind") | |
| call_id = str(check.get("call_id", "")) | |
| if kind == "duplicate_tool_result": | |
| minimum = int(check.get("min_count", 2)) | |
| actual = counts.get(call_id, 0) | |
| passed = actual >= minimum | |
| messages.append(f"duplicate_tool_result {call_id}: observed={actual} expected>={minimum}") | |
| ok = ok and passed | |
| elif kind == "duplicate_tool_result_in_request": | |
| minimum = int(check.get("min_count", 2)) | |
| actual = max_tool_result_count_in_one_request(requests, call_id) | |
| passed = actual >= minimum | |
| messages.append(f"duplicate_tool_result_in_request {call_id or '<any>'}: observed={actual} expected>={minimum}") | |
| ok = ok and passed | |
| elif kind == "tool_result": | |
| actual = counts.get(call_id, 0) | |
| passed = actual >= int(check.get("min_count", 1)) | |
| messages.append(f"tool_result {call_id}: observed={actual} expected>={int(check.get('min_count', 1))}") | |
| ok = ok and passed | |
| elif kind == "tool_result_payload_contains": | |
| needle = str(check.get("text", "")) | |
| actual = any(result["call_id"] == call_id and needle in result.get("payload_text", "") for result in results) | |
| messages.append(f"tool_result_payload_contains {call_id} {needle!r}: observed={actual}") | |
| ok = ok and actual | |
| elif kind == "any_tool_result_after_signal": | |
| minimum = int(check.get("min_count", 1)) | |
| actual = len(result_ids) | |
| passed = actual >= minimum | |
| messages.append(f"tool_result_after_signal: observed={actual} expected>={minimum}") | |
| ok = ok and passed | |
| elif kind == "late_stream_events_after_stop": | |
| minimum = int(check.get("min_count", 1)) | |
| actual = len(late_events) | |
| passed = actual >= minimum | |
| messages.append(f"late_stream_events_after_stop: observed={actual} expected>={minimum}") | |
| ok = ok and passed | |
| elif kind == "pid_file_alive": | |
| raw_path = str(check.get("path", "")) | |
| rel = Path(raw_path) | |
| if rel.is_absolute() or ".." in rel.parts: | |
| raise ValueError(f"unsafe side-effect path: {raw_path}") | |
| pid_file = side_effects / rel | |
| pid_text = pid_file.read_text(encoding="utf-8").strip() if pid_file.exists() else "" | |
| alive_file = side_effects / str(check.get("alive_file", case.get("alive_file", "child.alive"))) | |
| if alive_file.exists(): | |
| alive_text = alive_file.read_text(encoding="utf-8").strip().splitlines() | |
| actual = bool(alive_text and alive_text[0] == "alive") | |
| if len(alive_text) > 1 and not pid_text: | |
| pid_text = alive_text[1] | |
| else: | |
| try: | |
| pid = int(pid_text) | |
| except ValueError: | |
| pid = -1 | |
| actual = is_pid_alive(pid) | |
| messages.append(f"pid_file_alive {raw_path}: pid={pid_text or '<missing>'} observed={actual}") | |
| ok = ok and actual | |
| elif kind == "side_effect_file_contains": | |
| raw_path = str(check.get("path", "")) | |
| rel = Path(raw_path) | |
| if rel.is_absolute() or ".." in rel.parts: | |
| raise ValueError(f"unsafe side-effect path: {raw_path}") | |
| needle = str(check.get("text", "")) | |
| path = side_effects / rel | |
| content = path.read_text(encoding="utf-8", errors="replace") if path.exists() else "" | |
| actual = needle in content | |
| messages.append(f"side_effect_file_contains {raw_path} {needle!r}: observed={actual}") | |
| ok = ok and actual | |
| elif kind == "side_effect_file_occurrences": | |
| raw_path = str(check.get("path", "")) | |
| rel = Path(raw_path) | |
| if rel.is_absolute() or ".." in rel.parts: | |
| raise ValueError(f"unsafe side-effect path: {raw_path}") | |
| needle = str(check.get("text", "")) | |
| minimum = int(check.get("min_count", 1)) | |
| path = side_effects / rel | |
| content = path.read_text(encoding="utf-8", errors="replace") if path.exists() else "" | |
| actual = content.count(needle) if needle else 0 | |
| passed = actual >= minimum | |
| messages.append(f"side_effect_file_occurrences {raw_path} {needle!r}: observed={actual} expected>={minimum}") | |
| ok = ok and passed | |
| elif kind == "provider_requests_at_least": | |
| minimum = int(check.get("min_count", 1)) | |
| actual = len(requests) | |
| passed = actual >= minimum | |
| messages.append(f"provider_requests_at_least: observed={actual} expected>={minimum}") | |
| ok = ok and passed | |
| elif kind == "tool_result_bytes_at_least": | |
| minimum = int(check.get("min_bytes", 1)) | |
| passed = total_result_bytes >= minimum | |
| messages.append(f"tool_result_bytes_at_least: observed={total_result_bytes} expected>={minimum}") | |
| ok = ok and passed | |
| elif kind == "large_tool_results_at_least": | |
| threshold = int(check.get("min_bytes", 200_000)) | |
| minimum = int(check.get("min_count", 1)) | |
| actual = sum( | |
| 1 | |
| for result in results | |
| if len(str(result.get("payload_text", "")).encode("utf-8", errors="ignore")) >= threshold | |
| ) | |
| passed = actual >= minimum | |
| messages.append(f"large_tool_results_at_least: observed={actual} expected>={minimum}") | |
| ok = ok and passed | |
| elif kind == "closed_mid_stream_at_least": | |
| minimum = int(check.get("min_count", 1)) | |
| passed = closed_mid_stream >= minimum | |
| messages.append(f"closed_mid_stream_at_least: observed={closed_mid_stream} expected>={minimum}") | |
| ok = ok and passed | |
| elif kind == "nonzero_or_timeout": | |
| passed = bool(proc.returncode not in (0, None) or timed_out) | |
| messages.append(f"nonzero_or_timeout: returncode={proc.returncode} timed_out={timed_out}") | |
| ok = ok and passed | |
| elif kind == "timeout_or_signal": | |
| passed = bool(timed_out or (proc.returncode is not None and proc.returncode < 0) or proc.returncode == 124) | |
| messages.append(f"timeout_or_signal: returncode={proc.returncode} timed_out={timed_out}") | |
| ok = ok and passed | |
| else: | |
| ok = False | |
| messages.append(f"unknown check: {kind}") | |
| return ok, messages | |
| def main() -> int: | |
| parser = argparse.ArgumentParser() | |
| parser.add_argument("case", type=Path, nargs="?") | |
| parser.add_argument("--skip-build", action="store_true") | |
| args = parser.parse_args() | |
| if args.case is None: | |
| if EMBEDDED_CASE is None: | |
| parser.error("case JSON is required unless this file embeds a case") | |
| case = EMBEDDED_CASE | |
| else: | |
| case = json.loads(args.case.read_text(encoding="utf-8")) | |
| runs = expanded_case_runs(case) | |
| if not args.skip_build: | |
| built: set[str] = set() | |
| for run in runs: | |
| image = derive_image(run) | |
| if image not in built: | |
| build_image(run, image) | |
| built.add(image) | |
| all_ok = True | |
| for run in runs: | |
| image = derive_image(run) | |
| state = ProviderState(run) | |
| server = ProviderServer(("127.0.0.1", 0), state) | |
| thread = threading.Thread(target=server.serve_forever, daemon=True) | |
| thread.start() | |
| _host, port = server.server_address | |
| base_url = f"http://{provider_host_for_container()}:{port}" | |
| try: | |
| with temporary_workspace() as tmp: | |
| workspace = Path(tmp) | |
| side_effects = workspace / "side-effects" | |
| side_effects.mkdir(parents=True, exist_ok=True) | |
| extra_env = prepare_workspace(workspace, str(run["target"]), base_url, run) | |
| proc = run_target(run, image, base_url, workspace, side_effects, extra_env) | |
| requests = state.snapshot_requests() | |
| ok, messages = evaluate(run, requests, proc, side_effects) | |
| all_ok = all_ok and ok | |
| print("\n=== Reproduction summary ===") | |
| print(f"case={case.get('slug', '<embedded>')} run={run.get('run_name', run['target'])}") | |
| print(f"target={run['target']} image={image}") | |
| print(f"process_exit={proc.returncode}") | |
| for message in messages: | |
| print(message) | |
| if proc.stdout: | |
| print("stdout_tail=" + proc.stdout[-1000:].decode("utf-8", errors="replace")) | |
| if proc.stderr: | |
| print("stderr_tail=" + proc.stderr[-1000:].decode("utf-8", errors="replace")) | |
| print("RUN_REPRODUCED" if ok else "RUN_NOT_REPRODUCED") | |
| finally: | |
| server.shutdown() | |
| server.server_close() | |
| thread.join(timeout=2) | |
| print("REPRODUCED" if all_ok else "NOT_REPRODUCED") | |
| return 0 if all_ok else 1 | |
| if __name__ == "__main__": | |
| raise SystemExit(main()) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment