Last active
June 21, 2026 12:48
-
-
Save N0zoM1z0/b6b7daaf046c5d51345a54f886b11b61 to your computer and use it in GitHub Desktop.
Standalone reproducer for opencode-interrupt-child-late-effect
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/usr/bin/env python3 | |
| """Standalone vendor reproducer for CLI provider-stream bugs. | |
| Usage: | |
| python3 reproducer.py case.json | |
| The script intentionally imports only Python standard-library modules. It | |
| starts a local mock provider, runs the target CLI in Docker, records provider | |
| requests, and evaluates simple case assertions. | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| import os | |
| from pathlib import Path | |
| import shlex | |
| import shutil | |
| import signal | |
| import subprocess | |
| import sys | |
| import tempfile | |
| import threading | |
| import time | |
| from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer | |
| from typing import Any | |
| PROMPT = "Briefly inspect this directory and use the requested tool if needed." | |
| DOCKER_PATH = "/home/runner/.npm-global/bin:/home/runner/.cargo/bin:/usr/local/cargo/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" | |
| DEFAULT_DOCKER_CPUS = "2" | |
| DEFAULT_DOCKER_MEMORY = "4g" | |
| EMBEDDED_CASE: dict[str, Any] | None = json.loads(r''' | |
| { | |
| "alive_file": "interrupt-child.alive", | |
| "checks": [ | |
| { | |
| "kind": "provider_requests_at_least", | |
| "min_count": 1 | |
| }, | |
| { | |
| "kind": "side_effect_file_exists", | |
| "path": "interrupt-child.pid" | |
| }, | |
| { | |
| "kind": "timeout_or_signal" | |
| } | |
| ], | |
| "image": "vendor-repro-opencode:1.17.9", | |
| "pid_file": "interrupt-child.pid", | |
| "side_effect_probe": true, | |
| "signal_after_ms": 800, | |
| "slug": "opencode-interrupt-child-late-effect", | |
| "steps": [ | |
| { | |
| "provider.time.delay_ms": { | |
| "value": 2000 | |
| } | |
| }, | |
| { | |
| "provider.stream.message_start": {} | |
| }, | |
| { | |
| "provider.stream.tool_call_start": { | |
| "id": "interrupt_child_process", | |
| "name": "Bash" | |
| } | |
| }, | |
| { | |
| "provider.stream.tool_args_delta": { | |
| "delta": "{\"command\":\"python3 -c \\\"import os, pathlib, subprocess, time; root=pathlib.Path(os.environ['REPRO_SIDE_EFFECT_ROOT']); root.mkdir(parents=True, exist_ok=True); p=subprocess.Popen(['sleep','60'], start_new_session=True); (root/'interrupt-child.pid').write_text(str(p.pid)); time.sleep(3)\\\"\"}", | |
| "id": "interrupt_child_process" | |
| } | |
| }, | |
| { | |
| "provider.stream.tool_call_stop": { | |
| "id": "interrupt_child_process" | |
| } | |
| }, | |
| { | |
| "provider.stream.message_stop": {} | |
| } | |
| ], | |
| "target": "opencode", | |
| "timeout_s": 45, | |
| "version": "1.17.9", | |
| "workspace_files": { | |
| "src/app.js": "console.log('vendor repro');\n" | |
| } | |
| } | |
| ''') | |
| def dockerfile_for_target(target: str, version: str) -> str: | |
| if target == "qqcode": | |
| return f"""FROM python:3.12-slim | |
| ENV DEBIAN_FRONTEND=noninteractive | |
| RUN apt-get update && apt-get install -y --no-install-recommends \\ | |
| bash git curl ca-certificates procps lsof ripgrep \\ | |
| && rm -rf /var/lib/apt/lists/* | |
| RUN python3 -m pip install --no-cache-dir qqcode=={version} | |
| WORKDIR /workspace | |
| ENTRYPOINT ["/usr/bin/env"] | |
| CMD ["bash"] | |
| """ | |
| if target == "mini_swe_agent": | |
| return f"""FROM python:3.12-slim | |
| ENV DEBIAN_FRONTEND=noninteractive | |
| RUN apt-get update && apt-get install -y --no-install-recommends \\ | |
| bash git curl ca-certificates procps lsof ripgrep \\ | |
| && rm -rf /var/lib/apt/lists/* | |
| RUN python3 -m pip install --no-cache-dir mini-swe-agent=={version} | |
| WORKDIR /workspace | |
| ENTRYPOINT ["/usr/bin/env"] | |
| CMD ["bash"] | |
| """ | |
| if target == "claurst": | |
| return f"""FROM debian:trixie-slim | |
| ENV DEBIAN_FRONTEND=noninteractive | |
| RUN apt-get update && apt-get install -y --no-install-recommends \\ | |
| bash python3 curl ca-certificates procps lsof ripgrep tar gzip libasound2t64 \\ | |
| && rm -rf /var/lib/apt/lists/* | |
| RUN set -eux; \\ | |
| curl -fsSL --retry 3 --connect-timeout 20 --max-time 180 \\ | |
| https://github.com/Kuberwastaken/claurst/releases/download/v{version}/claurst-linux-x86_64.tar.gz \\ | |
| -o /tmp/claurst.tgz; \\ | |
| tar -xzf /tmp/claurst.tgz -C /usr/local/bin; \\ | |
| chmod +x /usr/local/bin/claurst; \\ | |
| rm -f /tmp/claurst.tgz | |
| WORKDIR /workspace | |
| ENTRYPOINT ["/usr/bin/env"] | |
| CMD ["bash"] | |
| """ | |
| if target == "picocode": | |
| return f"""FROM debian:trixie-slim | |
| ENV DEBIAN_FRONTEND=noninteractive | |
| RUN apt-get update && apt-get install -y --no-install-recommends \\ | |
| bash python3 curl ca-certificates procps lsof ripgrep tar gzip \\ | |
| && rm -rf /var/lib/apt/lists/* | |
| RUN set -eux; \\ | |
| curl -fsSL --retry 3 --connect-timeout 20 --max-time 180 \\ | |
| https://github.com/jondot/picocode/releases/download/v{version}/picocode-linux-x86_64.tar.gz \\ | |
| -o /tmp/picocode.tgz; \\ | |
| tar -xzf /tmp/picocode.tgz -C /usr/local/bin; \\ | |
| chmod +x /usr/local/bin/picocode; \\ | |
| rm -f /tmp/picocode.tgz | |
| WORKDIR /workspace | |
| ENTRYPOINT ["/usr/bin/env"] | |
| CMD ["bash"] | |
| """ | |
| if target == "free_code": | |
| return f"""FROM oven/bun:1.3-debian | |
| ENV DEBIAN_FRONTEND=noninteractive | |
| RUN apt-get update && apt-get install -y --no-install-recommends \\ | |
| bash git curl ca-certificates python3 procps lsof ripgrep \\ | |
| && rm -rf /var/lib/apt/lists/* | |
| RUN set -eux; \\ | |
| git clone --depth 1 https://github.com/freecodexyz/free-code.git /opt/free-code; \\ | |
| cd /opt/free-code; \\ | |
| git fetch --depth 1 origin {version}; \\ | |
| git checkout --detach {version}; \\ | |
| bun install --frozen-lockfile; \\ | |
| bun run build:dev:full; \\ | |
| ln -sf /opt/free-code/cli-dev /usr/local/bin/free-code | |
| WORKDIR /workspace | |
| ENTRYPOINT ["/usr/bin/env"] | |
| CMD ["bash"] | |
| """ | |
| if target == "crab_code": | |
| return f"""FROM rust:1-bookworm | |
| ENV DEBIAN_FRONTEND=noninteractive | |
| RUN apt-get update && apt-get install -y --no-install-recommends \\ | |
| bash git curl ca-certificates python3 procps lsof ripgrep pkg-config libssl-dev \\ | |
| && rm -rf /var/lib/apt/lists/* | |
| RUN set -eux; \\ | |
| git clone --depth 1 https://github.com/lingcoder/crab-code.git /opt/crab-code; \\ | |
| cd /opt/crab-code; \\ | |
| git fetch --depth 1 origin {version}; \\ | |
| git checkout --detach {version}; \\ | |
| cargo build --release --bin crab; \\ | |
| cp target/release/crab /usr/local/bin/crab; \\ | |
| chmod +x /usr/local/bin/crab | |
| WORKDIR /workspace | |
| ENTRYPOINT ["/usr/bin/env"] | |
| CMD ["bash"] | |
| """ | |
| if target == "hermes_agent": | |
| return f"""FROM node:24-bookworm | |
| ENV DEBIAN_FRONTEND=noninteractive | |
| ENV NPM_CONFIG_PREFIX=/home/runner/.npm-global | |
| ENV PIP_BREAK_SYSTEM_PACKAGES=1 | |
| ENV PATH=/home/runner/.npm-global/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin | |
| RUN apt-get update && apt-get install -y --no-install-recommends \\ | |
| python3 python3-pip python3-venv git curl ca-certificates procps lsof ripgrep passwd \\ | |
| && rm -rf /var/lib/apt/lists/* | |
| RUN set -eux; \\ | |
| useradd -m -d /home/runner -s /bin/bash runner; \\ | |
| mkdir -p /home/runner/.npm-global; \\ | |
| chown -R runner:runner /home/runner | |
| RUN npm install -g hermes-agent@{version} && chmod -R a+rx /home/runner | |
| WORKDIR /workspace | |
| ENTRYPOINT ["/usr/bin/env"] | |
| CMD ["bash"] | |
| """ | |
| if target == "zeroclaw": | |
| return f"""FROM debian:trixie-slim | |
| ENV DEBIAN_FRONTEND=noninteractive | |
| ENV PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin | |
| RUN apt-get update && apt-get install -y --no-install-recommends \\ | |
| python3 curl ca-certificates \\ | |
| && rm -rf /var/lib/apt/lists/* | |
| RUN set -eux; \\ | |
| curl -fsSL --retry 3 --connect-timeout 20 --max-time 180 \\ | |
| https://github.com/zeroclaw-labs/zeroclaw/releases/download/v{version}/zeroclaw-x86_64-unknown-linux-gnu.tar.gz \\ | |
| -o /tmp/zeroclaw.tgz; \\ | |
| tar -xzf /tmp/zeroclaw.tgz -C /usr/local/bin zeroclaw; \\ | |
| chmod +x /usr/local/bin/zeroclaw; \\ | |
| rm -f /tmp/zeroclaw.tgz | |
| WORKDIR /workspace | |
| ENTRYPOINT ["/usr/bin/env"] | |
| CMD ["bash"] | |
| """ | |
| common = """FROM node:24-bookworm | |
| ARG RUNNER_UID=1000 | |
| ARG RUNNER_GID=1000 | |
| ENV DEBIAN_FRONTEND=noninteractive | |
| ENV NPM_CONFIG_PREFIX=/home/runner/.npm-global | |
| ENV PATH=/home/runner/.npm-global/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin | |
| RUN apt-get update && apt-get install -y --no-install-recommends \\ | |
| python3 git curl ca-certificates procps lsof ripgrep passwd \\ | |
| && rm -rf /var/lib/apt/lists/* | |
| RUN set -eux; \\ | |
| if ! getent group "${RUNNER_GID}" >/dev/null; then groupadd -g "${RUNNER_GID}" runner; fi; \\ | |
| if ! getent passwd "${RUNNER_UID}" >/dev/null; then \\ | |
| useradd -m -u "${RUNNER_UID}" -g "${RUNNER_GID}" -d /home/runner -s /bin/bash runner; \\ | |
| else \\ | |
| existing_user="$(getent passwd "${RUNNER_UID}" | cut -d: -f1)"; \\ | |
| usermod -d /home/runner "${existing_user}" || true; \\ | |
| fi; \\ | |
| mkdir -p /home/runner/.npm-global; \\ | |
| chown -R "${RUNNER_UID}:${RUNNER_GID}" /home/runner; \\ | |
| chmod -R a+rx /home/runner | |
| """ | |
| if target == "claude": | |
| install = f"RUN npm install -g @anthropic-ai/claude-code@{version} && chmod -R a+rx /home/runner\n" | |
| elif target == "codex": | |
| install = f"RUN npm install -g @openai/codex@{version} && chmod -R a+rx /home/runner\n" | |
| elif target == "every_code": | |
| install = f"RUN npm install -g @just-every/code@{version} && chmod -R a+rx /home/runner\n" | |
| elif target == "gitagent": | |
| install = f"RUN npm install -g @open-gitagent/gitagent@{version} && chmod -R a+rx /home/runner\n" | |
| elif target == "open_codex": | |
| install = f"RUN npm install -g open-codex@{version} && chmod -R a+rx /home/runner\n" | |
| elif target == "qwen": | |
| install = f"RUN npm install -g @qwen-code/qwen-code@{version} && chmod -R a+rx /home/runner\n" | |
| elif target == "gemini": | |
| install = f"RUN npm install -g @google/gemini-cli@{version} && chmod -R a+rx /home/runner\n" | |
| elif target == "forgecode": | |
| install = f"RUN npm install -g @antinomyhq/forge@{version} && chmod -R a+rx /home/runner\n" | |
| elif target == "groq_code_cli": | |
| install = f"RUN npm install -g groq-code-cli@{version} && chmod -R a+rx /home/runner\n" | |
| elif target == "kilo_code": | |
| install = f"RUN npm install -g @kilocode/cli@{version} && chmod -R a+rx /home/runner\n" | |
| elif target == "kode_cli": | |
| install = f"RUN npm install -g @shareai-lab/kode@{version} && chmod -R a+rx /home/runner\n" | |
| elif target == "letta_code": | |
| install = f"RUN npm install -g @letta-ai/letta-code@{version} && chmod -R a+rx /home/runner\n" | |
| elif target == "mini_kode": | |
| install = f"RUN npm install -g mini-kode@{version} && chmod -R a+rx /home/runner\n" | |
| elif target == "neovate_code": | |
| install = f"RUN npm install -g @neovate/code@{version} && chmod -R a+rx /home/runner\n" | |
| elif target == "pi": | |
| install = f"RUN npm install -g @earendil-works/pi-coding-agent@{version} && chmod -R a+rx /home/runner\n" | |
| elif target == "crush": | |
| install = f"RUN npm install -g @charmland/crush@{version} && chmod -R a+rx /home/runner\n" | |
| elif target == "goose": | |
| install = ( | |
| "RUN curl -fsSL --retry 3 --connect-timeout 20 --max-time 180 " | |
| "https://github.com/aaif-goose/goose/releases/download/stable/download_cli.sh " | |
| f"| GOOSE_VERSION=v{version} CONFIGURE=false GOOSE_BIN_DIR=/usr/local/bin bash\n" | |
| ) | |
| elif target == "cline": | |
| install = ( | |
| f"RUN npm install -g cline@{version} " | |
| f"&& cline_root=\"$(npm root -g)/@cline/cli-linux-x64\" " | |
| "&& if [ ! -x \"$cline_root/bin/cline\" ]; then " | |
| "tmp=\"$(mktemp -d)\"; " | |
| f"curl -fsSL --retry 3 https://registry.npmjs.org/@cline/cli-linux-x64/-/cli-linux-x64-{version}.tgz -o \"$tmp/cline.tgz\"; " | |
| "tar -xzf \"$tmp/cline.tgz\" -C \"$tmp\"; " | |
| "mkdir -p \"$cline_root/bin\"; " | |
| "cp \"$tmp/package/bin/cline\" \"$cline_root/bin/cline\"; " | |
| "chmod +x \"$cline_root/bin/cline\"; " | |
| "fi " | |
| "&& chmod -R a+rx /home/runner\n" | |
| ) | |
| elif target == "opencode": | |
| install = f"RUN npm install -g opencode-ai@{version} && chmod -R a+rx /home/runner\n" | |
| elif target == "continue": | |
| install = f"RUN npm install -g @continuedev/cli@{version} && chmod -R a+rx /home/runner\n" | |
| elif target == "openharness": | |
| install = f"RUN npm install -g @zhijiewang/openharness@{version} && chmod -R a+rx /home/runner\n" | |
| else: | |
| raise ValueError(f"unsupported target for Dockerfile: {target}") | |
| return common + install + 'WORKDIR /workspace\nENTRYPOINT ["/usr/bin/env"]\nCMD ["bash"]\n' | |
| class ProviderState: | |
| def __init__(self, case: dict[str, Any]) -> None: | |
| self.case = case | |
| self.requests: list[dict[str, Any]] = [] | |
| self.lock = threading.Lock() | |
| def append_request(self, entry: dict[str, Any]) -> None: | |
| with self.lock: | |
| self.requests.append(entry) | |
| def request_count(self) -> int: | |
| with self.lock: | |
| return len(self.requests) | |
| def snapshot_requests(self) -> list[dict[str, Any]]: | |
| with self.lock: | |
| return list(self.requests) | |
| class ProviderServer(ThreadingHTTPServer): | |
| def __init__(self, address: tuple[str, int], state: ProviderState) -> None: | |
| self.state = state | |
| super().__init__(address, ProviderHandler) | |
| class ProviderHandler(BaseHTTPRequestHandler): | |
| server: ProviderServer | |
| def log_message(self, fmt: str, *args: Any) -> None: | |
| return None | |
| def do_GET(self) -> None: | |
| if self.path in {"/health", "/__repro/health"}: | |
| self._json(200, {"ok": True}) | |
| return | |
| if self.path in {"/v1/models", "/v1/anthropic/models"}: | |
| self._json( | |
| 200, | |
| { | |
| "object": "list", | |
| "data": [ | |
| {"id": "gpt-4", "object": "model"}, | |
| {"id": "claude-3-5-sonnet-20241022", "type": "model"}, | |
| {"id": "claude-sonnet-4-5", "type": "model"}, | |
| ], | |
| }, | |
| ) | |
| return | |
| self._json(404, {"error": "not found"}) | |
| def do_POST(self) -> None: | |
| started = time.time() | |
| body = self._read_json() | |
| requests_before = self.server.state.request_count() | |
| response = select_response(self.server.state.case, body, requests_before) | |
| entry = { | |
| "ts": started, | |
| "path": self.path, | |
| "body": body, | |
| "response_events": [], | |
| "status": response.get("status", 200), | |
| "closed_mid_stream": False, | |
| } | |
| self.server.state.append_request(entry) | |
| if response.get("kind") == "error": | |
| self._json(int(response.get("status", 500)), {"error": response.get("error", {})}) | |
| return | |
| wire = wire_for_path(self.path) | |
| steps = response.get("steps", []) | |
| if wire == "openai_chat" and body.get("stream") is not True: | |
| self._json(200, encode_openai_chat_json(steps, str(body.get("model", "")) or "gpt-4")) | |
| return | |
| if wire == "openai_responses" and body.get("stream") is not True: | |
| self._json(200, encode_openai_responses_json(steps, str(body.get("model", "")) or "gpt-4")) | |
| return | |
| events = encode_stream(self.server.state.case, steps, wire, str(body.get("model", ""))) | |
| close_after = response.get("close_after_events") | |
| self._sse(entry, events, int(close_after) if close_after is not None else None) | |
| def _read_json(self) -> dict[str, Any]: | |
| length = int(self.headers.get("content-length") or 0) | |
| if length <= 0: | |
| return {} | |
| raw = self.rfile.read(length) | |
| try: | |
| return json.loads(raw.decode("utf-8")) | |
| except json.JSONDecodeError: | |
| return {"_raw": raw.decode("utf-8", errors="replace")} | |
| def _json(self, status: int, payload: dict[str, Any]) -> None: | |
| raw = json.dumps(payload, sort_keys=True).encode("utf-8") | |
| self.send_response(status) | |
| self.send_header("content-type", "application/json") | |
| self.send_header("content-length", str(len(raw))) | |
| self.end_headers() | |
| self.wfile.write(raw) | |
| def _sse(self, entry: dict[str, Any], events: list[bytes], close_after_events: int | None = None) -> None: | |
| self.send_response(200) | |
| self.send_header("content-type", "text/event-stream") | |
| self.send_header("cache-control", "no-cache") | |
| self.end_headers() | |
| seen_stop = False | |
| sent = 0 | |
| for event in events: | |
| if event.startswith(b": delay-ms:"): | |
| try: | |
| time.sleep(int(event.decode().split(":", 2)[2]) / 1000.0) | |
| except (IndexError, ValueError): | |
| pass | |
| continue | |
| sent += 1 | |
| decoded = event.decode("utf-8", errors="replace") | |
| event_entry = { | |
| "raw": decoded[:500], | |
| "after_stop": seen_stop, | |
| "is_stop": is_stop_event(decoded), | |
| "broken_pipe": False, | |
| } | |
| entry["response_events"].append(event_entry) | |
| try: | |
| self.wfile.write(event) | |
| self.wfile.flush() | |
| except BrokenPipeError: | |
| entry["broken_pipe"] = True | |
| event_entry["broken_pipe"] = True | |
| return | |
| if event_entry["is_stop"]: | |
| seen_stop = True | |
| if close_after_events is not None and sent >= close_after_events: | |
| entry["closed_mid_stream"] = True | |
| self.close_connection = True | |
| return | |
| def select_response(case: dict[str, Any], body: dict[str, Any], requests_before: int) -> dict[str, Any]: | |
| if has_tool_result(body): | |
| sequence = case.get("provider_followup_sequence") | |
| if isinstance(sequence, list): | |
| index = max(0, requests_before - 1) | |
| if index < len(sequence) and isinstance(sequence[index], dict): | |
| followup = sequence[index] | |
| if followup.get("kind") == "error": | |
| return followup | |
| return { | |
| "kind": "stream", | |
| "steps": adapt_steps_to_tools(followup.get("steps", []), body, requests_before), | |
| "close_after_events": followup.get("close_after_events"), | |
| } | |
| followup = case.get("provider_followup") | |
| if isinstance(followup, dict): | |
| if followup.get("kind") == "error": | |
| return followup | |
| if isinstance(followup.get("steps"), list): | |
| return { | |
| "kind": "stream", | |
| "steps": adapt_steps_to_tools(followup["steps"], body, requests_before), | |
| "close_after_events": followup.get("close_after_events"), | |
| } | |
| retry = case.get("provider_retry") | |
| if isinstance(retry, dict): | |
| after_requests = int(retry.get("after_requests", 1)) | |
| if requests_before >= after_requests: | |
| if retry.get("kind") == "error": | |
| return retry | |
| if isinstance(retry.get("steps"), list): | |
| return { | |
| "kind": "stream", | |
| "steps": adapt_steps_to_tools(retry["steps"], body, requests_before), | |
| "close_after_events": retry.get("close_after_events"), | |
| } | |
| return { | |
| "kind": "stream", | |
| "steps": adapt_steps_to_tools(case.get("steps", []), body, requests_before), | |
| "close_after_events": case.get("close_after_events"), | |
| } | |
| def render_step_templates(value: Any, request_index: int) -> Any: | |
| if isinstance(value, str): | |
| return ( | |
| value.replace("{{request_index}}", str(request_index)) | |
| .replace("{{request_number}}", str(request_index + 1)) | |
| ) | |
| if isinstance(value, list): | |
| return [render_step_templates(item, request_index) for item in value] | |
| if isinstance(value, dict): | |
| return {key: render_step_templates(item, request_index) for key, item in value.items()} | |
| return value | |
| def adapt_steps_to_tools(steps: list[dict[str, Any]], body: dict[str, Any], request_index: int = 0) -> list[dict[str, Any]]: | |
| advertised = advertised_tool_schemas(body) | |
| if not advertised: | |
| return [render_step_templates(step, request_index) for step in steps] | |
| shell_tool = choose_shell_tool(advertised) | |
| if not shell_tool: | |
| return [render_step_templates(step, request_index) for step in steps] | |
| shell_name, shell_arg = shell_tool | |
| shell_call_ids: set[str] = set() | |
| out: list[dict[str, Any]] = [] | |
| changed = False | |
| for action, payload in normalized_steps(steps): | |
| payload = render_step_templates(dict(payload), request_index) | |
| if action == "provider.stream.tool_call_start": | |
| name = str(payload.get("name", "")) | |
| if name in {"Bash", "Shell", "bash", "shell"} and name not in advertised: | |
| payload["name"] = shell_name | |
| shell_call_ids.add(str(payload.get("id", ""))) | |
| changed = True | |
| elif name == "Edit" and "Edit" not in advertised and "edit" in advertised: | |
| payload["name"] = "edit" | |
| changed = True | |
| elif action == "provider.stream.tool_args_delta" and str(payload.get("id", "")) in shell_call_ids: | |
| adapted = adapt_shell_arguments(str(payload.get("delta", "")), shell_arg, advertised.get(shell_name, set())) | |
| if adapted != payload.get("delta"): | |
| payload["delta"] = adapted | |
| changed = True | |
| out.append({action: payload}) | |
| return out if changed else steps | |
| def advertised_tool_schemas(body: dict[str, Any]) -> dict[str, set[str]]: | |
| schemas: dict[str, set[str]] = {} | |
| raw_tools = body.get("tools") | |
| if not isinstance(raw_tools, list): | |
| return schemas | |
| for item in raw_tools: | |
| if not isinstance(item, dict): | |
| continue | |
| declarations = item.get("function_declarations") or item.get("functionDeclarations") | |
| if isinstance(declarations, list): | |
| for declaration in declarations: | |
| if not isinstance(declaration, dict): | |
| continue | |
| name = str(declaration.get("name", "")) | |
| params = declaration.get("parameters") or {} | |
| properties = params.get("properties", {}) if isinstance(params, dict) else {} | |
| if name: | |
| schemas[name] = set(str(key) for key in properties) if isinstance(properties, dict) else set() | |
| continue | |
| function = item.get("function") | |
| if isinstance(function, dict): | |
| name = str(function.get("name", "")) | |
| params = function.get("parameters", {}) | |
| else: | |
| name = str(item.get("name", "")) | |
| params = item.get("parameters") or item.get("input_schema") or {} | |
| properties = params.get("properties", {}) if isinstance(params, dict) else {} | |
| if name: | |
| schemas[name] = set(str(key) for key in properties) if isinstance(properties, dict) else set() | |
| return schemas | |
| def choose_shell_tool(advertised: dict[str, set[str]]) -> tuple[str, str] | None: | |
| candidates = ( | |
| ("exec_command", "cmd"), | |
| ("execute_command", "command"), | |
| ("run_shell_command", "command"), | |
| ("run_commands", "commands"), | |
| ("terminal", "command"), | |
| ("run_command", "command"), | |
| ("shell", "command"), | |
| ("bash", "command"), | |
| ("Bash", "command"), | |
| ) | |
| for name, preferred_arg in candidates: | |
| if name not in advertised: | |
| continue | |
| properties = advertised[name] | |
| if preferred_arg in properties or not properties: | |
| return name, preferred_arg | |
| if "command" in properties: | |
| return name, "command" | |
| if "cmd" in properties: | |
| return name, "cmd" | |
| if "commands" in properties: | |
| return name, "commands" | |
| return None | |
| def adapt_shell_arguments(raw: str, arg_name: str, properties: set[str]) -> str: | |
| try: | |
| args = json.loads(raw) | |
| except json.JSONDecodeError: | |
| return raw | |
| if not isinstance(args, dict): | |
| return raw | |
| command = args.get("command") or args.get("cmd") | |
| if not isinstance(command, str): | |
| return raw | |
| adapted = dict(args) | |
| adapted.pop("command", None) | |
| adapted.pop("cmd", None) | |
| adapted[arg_name] = [command] if arg_name == "commands" else command | |
| if "description" in properties and "description" not in adapted: | |
| adapted["description"] = "standalone shell repro" | |
| return json.dumps(adapted, separators=(",", ":")) | |
| def wire_for_path(path: str) -> str: | |
| if ":streamGenerateContent" in path or ":generateContent" in path: | |
| return "gemini" | |
| if path.startswith("/v1/messages"): | |
| return "anthropic" | |
| if path == "/v1/responses": | |
| return "openai_responses" | |
| if path == "/v1/chat/completions": | |
| return "openai_chat" | |
| return "unknown" | |
| def is_stop_event(raw: str) -> bool: | |
| return ( | |
| "message_stop" in raw | |
| or "response.completed" in raw | |
| or '"finishReason": "STOP"' in raw | |
| or '"finishReason":"STOP"' in raw | |
| or raw.strip() == "data: [DONE]" | |
| ) | |
| def encode_stream(case: dict[str, Any], steps: list[dict[str, Any]], wire: str, model: str) -> list[bytes]: | |
| if wire == "anthropic": | |
| return encode_anthropic(steps, model or "claude-3-5-sonnet-20241022", bool(case.get("anthropic_full_input"))) | |
| if wire == "openai_responses": | |
| return encode_openai_responses(steps, model or "gpt-4") | |
| if wire == "openai_chat": | |
| return encode_openai_chat(steps, model or "gpt-4") | |
| if wire == "gemini": | |
| return encode_gemini(steps, model or "gemini-3-flash") | |
| return [sse("error", {"error": {"message": "unsupported path"}})] | |
| def normalized_steps(steps: list[dict[str, Any]]) -> list[tuple[str, dict[str, Any]]]: | |
| out = [] | |
| for item in steps: | |
| if not isinstance(item, dict) or len(item) != 1: | |
| raise ValueError(f"invalid step: {item!r}") | |
| action, payload = next(iter(item.items())) | |
| out.append((str(action), payload if isinstance(payload, dict) else {})) | |
| return out | |
| def tool_args_by_id(steps: list[dict[str, Any]]) -> dict[str, str]: | |
| out: dict[str, str] = {} | |
| for action, payload in normalized_steps(steps): | |
| if action == "provider.stream.tool_args_delta": | |
| call_id = str(payload.get("id", "")) | |
| out[call_id] = out.get(call_id, "") + str(payload.get("delta", "")) | |
| return out | |
| def encode_anthropic(steps: list[dict[str, Any]], model: str, full_input: bool = False) -> list[bytes]: | |
| events: list[bytes] = [] | |
| args_by_id = tool_args_by_id(steps) | |
| message_started = False | |
| next_index = 0 | |
| open_index: int | None = None | |
| index_by_id: dict[str, int] = {} | |
| def message_start() -> None: | |
| nonlocal message_started | |
| if message_started: | |
| return | |
| message_started = True | |
| events.append( | |
| sse( | |
| "message_start", | |
| { | |
| "type": "message_start", | |
| "message": { | |
| "id": "msg_repro", | |
| "type": "message", | |
| "role": "assistant", | |
| "model": model, | |
| "content": [], | |
| "stop_reason": None, | |
| "usage": {"input_tokens": 1, "output_tokens": 1}, | |
| }, | |
| }, | |
| ) | |
| ) | |
| def block_stop() -> None: | |
| nonlocal open_index | |
| if open_index is not None: | |
| events.append(sse("content_block_stop", {"type": "content_block_stop", "index": open_index})) | |
| open_index = None | |
| for action, payload in normalized_steps(steps): | |
| if action == "provider.stream.message_start": | |
| message_start() | |
| elif action == "provider.stream.tool_call_start": | |
| message_start() | |
| block_stop() | |
| call_id = str(payload["id"]) | |
| name = str(payload["name"]) | |
| open_index = next_index | |
| index_by_id[call_id] = open_index | |
| next_index += 1 | |
| events.append( | |
| sse( | |
| "content_block_start", | |
| { | |
| "type": "content_block_start", | |
| "index": open_index, | |
| "content_block": { | |
| "type": "tool_use", | |
| "id": call_id, | |
| "name": name, | |
| "input": parse_json_object(args_by_id.get(call_id, "")) if full_input else {}, | |
| }, | |
| }, | |
| ) | |
| ) | |
| elif action == "provider.stream.tool_args_delta": | |
| call_id = str(payload.get("id", "")) | |
| if full_input and call_id in index_by_id: | |
| continue | |
| message_start() | |
| events.append( | |
| sse( | |
| "content_block_delta", | |
| { | |
| "type": "content_block_delta", | |
| "index": index_by_id.get(call_id, open_index or 0), | |
| "delta": {"type": "input_json_delta", "partial_json": str(payload.get("delta", ""))}, | |
| }, | |
| ) | |
| ) | |
| elif action == "provider.stream.tool_call_stop": | |
| block_stop() | |
| elif action == "provider.stream.message_stop": | |
| block_stop() | |
| events.append(sse("message_stop", {"type": "message_stop"})) | |
| elif action == "provider.time.delay_ms": | |
| events.append(f": delay-ms:{int(payload.get('value', 0))}\n\n".encode()) | |
| if message_started and not any(b"message_stop" in event for event in events): | |
| block_stop() | |
| events.append(sse("message_stop", {"type": "message_stop"})) | |
| return events | |
| def encode_openai_responses(steps: list[dict[str, Any]], model: str) -> list[bytes]: | |
| response_id = "resp_repro" | |
| events = [ | |
| sse( | |
| "response.created", | |
| { | |
| "type": "response.created", | |
| "response": { | |
| "id": response_id, | |
| "object": "response", | |
| "created_at": int(time.time()), | |
| "status": "in_progress", | |
| "model": model, | |
| "output": [], | |
| }, | |
| }, | |
| ), | |
| sse("response.in_progress", {"type": "response.in_progress", "response": {"id": response_id, "status": "in_progress"}}), | |
| ] | |
| output_index = 0 | |
| item_by_id: dict[str, dict[str, Any]] = {} | |
| args_by_id: dict[str, str] = {} | |
| for action, payload in normalized_steps(steps): | |
| if action == "provider.stream.tool_call_start": | |
| call_id = str(payload["id"]) | |
| item = { | |
| "id": "fc_" + call_id, | |
| "type": "function_call", | |
| "call_id": call_id, | |
| "name": str(payload["name"]), | |
| "arguments": "", | |
| "status": "in_progress", | |
| } | |
| item_by_id[call_id] = item | |
| args_by_id.setdefault(call_id, "") | |
| events.append( | |
| sse( | |
| "response.output_item.added", | |
| { | |
| "type": "response.output_item.added", | |
| "output_index": output_index, | |
| "item": item, | |
| }, | |
| ) | |
| ) | |
| elif action == "provider.stream.tool_args_delta": | |
| call_id = str(payload.get("id", "")) | |
| delta = str(payload.get("delta", "")) | |
| args_by_id[call_id] = args_by_id.get(call_id, "") + delta | |
| events.append( | |
| sse( | |
| "response.function_call_arguments.delta", | |
| { | |
| "type": "response.function_call_arguments.delta", | |
| "output_index": output_index, | |
| "delta": delta, | |
| }, | |
| ) | |
| ) | |
| elif action == "provider.stream.tool_call_stop": | |
| call_id = str(payload.get("id", "")) | |
| item = dict(item_by_id.get(call_id, {})) | |
| if item: | |
| item["status"] = "completed" | |
| item["arguments"] = args_by_id.get(call_id, "") | |
| item_by_id[call_id] = item | |
| events.append( | |
| sse( | |
| "response.function_call_arguments.done", | |
| { | |
| "type": "response.function_call_arguments.done", | |
| "output_index": output_index, | |
| "arguments": args_by_id.get(call_id, ""), | |
| }, | |
| ) | |
| ) | |
| events.append( | |
| sse( | |
| "response.output_item.done", | |
| { | |
| "type": "response.output_item.done", | |
| "output_index": output_index, | |
| "item": item, | |
| }, | |
| ) | |
| ) | |
| output_index += 1 | |
| elif action == "provider.stream.message_stop": | |
| events.append( | |
| sse( | |
| "response.completed", | |
| { | |
| "type": "response.completed", | |
| "response": { | |
| "id": response_id, | |
| "object": "response", | |
| "status": "completed", | |
| "model": model, | |
| "output": list(item_by_id.values()), | |
| "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}, | |
| }, | |
| }, | |
| ) | |
| ) | |
| elif action == "provider.time.delay_ms": | |
| events.append(f": delay-ms:{int(payload.get('value', 0))}\n\n".encode()) | |
| if not any(b"response.completed" in event for event in events): | |
| events.append( | |
| sse( | |
| "response.completed", | |
| { | |
| "type": "response.completed", | |
| "response": { | |
| "id": response_id, | |
| "object": "response", | |
| "status": "completed", | |
| "model": model, | |
| "output": list(item_by_id.values()), | |
| "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}, | |
| }, | |
| }, | |
| ) | |
| ) | |
| return events | |
| def encode_openai_responses_json(steps: list[dict[str, Any]], model: str) -> dict[str, Any]: | |
| response_id = "resp_repro" | |
| output: list[dict[str, Any]] = [] | |
| args_by_id = tool_args_by_id(steps) | |
| for action, payload in normalized_steps(steps): | |
| if action != "provider.stream.tool_call_start": | |
| continue | |
| call_id = str(payload["id"]) | |
| output.append( | |
| { | |
| "id": "fc_" + call_id, | |
| "type": "function_call", | |
| "call_id": call_id, | |
| "name": str(payload["name"]), | |
| "arguments": args_by_id.get(call_id, ""), | |
| "status": "completed", | |
| } | |
| ) | |
| if not output: | |
| output.append( | |
| { | |
| "id": "msg_repro", | |
| "type": "message", | |
| "status": "completed", | |
| "role": "assistant", | |
| "content": [{"type": "output_text", "text": "done"}], | |
| } | |
| ) | |
| return { | |
| "id": response_id, | |
| "object": "response", | |
| "created_at": int(time.time()), | |
| "status": "completed", | |
| "model": model, | |
| "output": output, | |
| "usage": { | |
| "input_tokens": 1, | |
| "input_tokens_details": {"cached_tokens": 0}, | |
| "output_tokens": 1, | |
| "output_tokens_details": {"reasoning_tokens": 0}, | |
| "total_tokens": 2, | |
| }, | |
| } | |
| def encode_openai_chat(steps: list[dict[str, Any]], model: str) -> list[bytes]: | |
| events: list[bytes] = [] | |
| chat_id = "chatcmpl_repro" | |
| started = False | |
| next_index = 0 | |
| index_by_id: dict[str, int] = {} | |
| def start() -> None: | |
| nonlocal started | |
| if started: | |
| return | |
| started = True | |
| events.append( | |
| sse( | |
| "", | |
| { | |
| "id": chat_id, | |
| "object": "chat.completion.chunk", | |
| "created": int(time.time()), | |
| "model": model, | |
| "choices": [{"index": 0, "delta": {"role": "assistant"}, "finish_reason": None}], | |
| }, | |
| ) | |
| ) | |
| has_tool_calls = False | |
| for action, payload in normalized_steps(steps): | |
| if action == "provider.stream.tool_call_start": | |
| start() | |
| has_tool_calls = True | |
| call_id = str(payload["id"]) | |
| index_by_id[call_id] = next_index | |
| next_index += 1 | |
| events.append( | |
| sse( | |
| "", | |
| { | |
| "choices": [ | |
| { | |
| "index": 0, | |
| "delta": { | |
| "tool_calls": [ | |
| { | |
| "index": index_by_id[call_id], | |
| "id": call_id, | |
| "type": "function", | |
| "function": {"name": str(payload["name"]), "arguments": ""}, | |
| } | |
| ] | |
| }, | |
| "finish_reason": None, | |
| } | |
| ] | |
| }, | |
| ) | |
| ) | |
| elif action == "provider.stream.tool_args_delta": | |
| start() | |
| call_id = str(payload.get("id", "")) | |
| events.append( | |
| sse( | |
| "", | |
| { | |
| "choices": [ | |
| { | |
| "index": 0, | |
| "delta": { | |
| "tool_calls": [ | |
| { | |
| "index": index_by_id.get(call_id, 0), | |
| "function": {"arguments": str(payload.get("delta", ""))}, | |
| } | |
| ] | |
| }, | |
| "finish_reason": None, | |
| } | |
| ] | |
| }, | |
| ) | |
| ) | |
| elif action == "provider.stream.message_stop": | |
| start() | |
| events.append( | |
| sse( | |
| "", | |
| { | |
| "id": chat_id, | |
| "object": "chat.completion.chunk", | |
| "created": int(time.time()), | |
| "model": model, | |
| "choices": [ | |
| { | |
| "index": 0, | |
| "delta": {}, | |
| "finish_reason": "tool_calls" if has_tool_calls else "stop", | |
| } | |
| ], | |
| }, | |
| ) | |
| ) | |
| events.append(b"data: [DONE]\n\n") | |
| elif action == "provider.time.delay_ms": | |
| events.append(f": delay-ms:{int(payload.get('value', 0))}\n\n".encode()) | |
| if not events or events[-1] != b"data: [DONE]\n\n": | |
| start() | |
| events.append( | |
| sse( | |
| "", | |
| { | |
| "id": chat_id, | |
| "object": "chat.completion.chunk", | |
| "created": int(time.time()), | |
| "model": model, | |
| "choices": [ | |
| { | |
| "index": 0, | |
| "delta": {}, | |
| "finish_reason": "tool_calls" if has_tool_calls else "stop", | |
| } | |
| ], | |
| }, | |
| ) | |
| ) | |
| events.append(b"data: [DONE]\n\n") | |
| return events | |
| def encode_openai_chat_json(steps: list[dict[str, Any]], model: str) -> dict[str, Any]: | |
| tool_calls: list[dict[str, Any]] = [] | |
| args_by_id = tool_args_by_id(steps) | |
| for action, payload in normalized_steps(steps): | |
| if action != "provider.stream.tool_call_start": | |
| continue | |
| call_id = str(payload["id"]) | |
| tool_calls.append( | |
| { | |
| "id": call_id, | |
| "type": "function", | |
| "function": { | |
| "name": str(payload["name"]), | |
| "arguments": args_by_id.get(call_id, ""), | |
| }, | |
| } | |
| ) | |
| message: dict[str, Any] = {"role": "assistant", "content": None if tool_calls else "done"} | |
| finish_reason = "stop" | |
| if tool_calls: | |
| message["tool_calls"] = tool_calls | |
| finish_reason = "tool_calls" | |
| return { | |
| "id": "chatcmpl_repro", | |
| "object": "chat.completion", | |
| "created": int(time.time()), | |
| "model": model, | |
| "choices": [{"index": 0, "message": message, "finish_reason": finish_reason}], | |
| "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, | |
| } | |
| def encode_gemini(steps: list[dict[str, Any]], model: str) -> list[bytes]: | |
| events: list[bytes] = [] | |
| args_by_id = tool_args_by_id(steps) | |
| for action, payload in normalized_steps(steps): | |
| if action == "provider.stream.message_start": | |
| continue | |
| if action == "provider.stream.tool_call_start": | |
| call_id = str(payload["id"]) | |
| events.append( | |
| sse( | |
| "", | |
| { | |
| "modelVersion": model, | |
| "candidates": [ | |
| { | |
| "content": { | |
| "parts": [ | |
| { | |
| "functionCall": { | |
| "name": str(payload["name"]), | |
| "args": parse_json_object(args_by_id.get(call_id, "")), | |
| } | |
| } | |
| ], | |
| "role": "model", | |
| }, | |
| "finishReason": "STOP", | |
| "index": 0, | |
| } | |
| ], | |
| "usageMetadata": { | |
| "promptTokenCount": 1, | |
| "candidatesTokenCount": 1, | |
| "totalTokenCount": 2, | |
| }, | |
| }, | |
| ) | |
| ) | |
| elif action == "provider.stream.message_stop": | |
| events.append( | |
| sse( | |
| "", | |
| { | |
| "modelVersion": model, | |
| "candidates": [ | |
| { | |
| "content": {"parts": [{"text": ""}], "role": "model"}, | |
| "finishReason": "STOP", | |
| "index": 0, | |
| } | |
| ], | |
| "usageMetadata": { | |
| "promptTokenCount": 1, | |
| "candidatesTokenCount": 1, | |
| "totalTokenCount": 2, | |
| }, | |
| }, | |
| ) | |
| ) | |
| elif action == "provider.time.delay_ms": | |
| events.append(f": delay-ms:{int(payload.get('value', 0))}\n\n".encode()) | |
| if not events: | |
| events.append( | |
| sse( | |
| "", | |
| { | |
| "modelVersion": model, | |
| "candidates": [ | |
| { | |
| "content": {"parts": [{"text": "done"}], "role": "model"}, | |
| "finishReason": "STOP", | |
| "index": 0, | |
| } | |
| ], | |
| "usageMetadata": { | |
| "promptTokenCount": 1, | |
| "candidatesTokenCount": 1, | |
| "totalTokenCount": 2, | |
| }, | |
| }, | |
| ) | |
| ) | |
| return events | |
| def parse_json_object(value: str) -> dict[str, Any]: | |
| try: | |
| parsed = json.loads(value or "{}") | |
| except json.JSONDecodeError: | |
| return {} | |
| return parsed if isinstance(parsed, dict) else {} | |
| def sse(event: str, payload: dict[str, Any]) -> bytes: | |
| prefix = f"event: {event}\n" if event else "" | |
| return (prefix + "data: " + json.dumps(payload, sort_keys=True) + "\n\n").encode() | |
| def has_tool_result(value: Any) -> bool: | |
| if isinstance(value, list): | |
| return any(has_tool_result(item) for item in value) | |
| if not isinstance(value, dict): | |
| return False | |
| raw_type = str(value.get("type", "")).lower() | |
| role = str(value.get("role", "")).lower() | |
| if "functionResponse" in value or "function_response" in value: | |
| return True | |
| if raw_type in {"function_call_output", "tool_result", "tool_use_result", "tool_call_output"}: | |
| return True | |
| if role == "tool": | |
| return True | |
| return any(has_tool_result(item) for item in value.values()) | |
| def collect_tool_results(value: Any, out: list[dict[str, Any]]) -> None: | |
| if isinstance(value, list): | |
| for item in value: | |
| collect_tool_results(item, out) | |
| return | |
| if not isinstance(value, dict): | |
| return | |
| raw_type = str(value.get("type", "")).lower() | |
| role = str(value.get("role", "")).lower() | |
| function_response = value.get("functionResponse") or value.get("function_response") | |
| if isinstance(function_response, dict): | |
| call_id = str(function_response.get("id") or function_response.get("name") or "functionResponse") | |
| payload = function_response.get("response", function_response) | |
| out.append( | |
| { | |
| "call_id": call_id, | |
| "type": "functionResponse", | |
| "payload_text": json.dumps(payload, default=str, sort_keys=True), | |
| } | |
| ) | |
| result_like = raw_type in {"function_call_output", "tool_result", "tool_use_result", "tool_call_output"} or role == "tool" | |
| if result_like: | |
| call_id = "" | |
| for key in ("call_id", "tool_call_id", "tool_use_id"): | |
| if value.get(key): | |
| call_id = str(value[key]) | |
| break | |
| if call_id: | |
| payload = value.get("output") if "output" in value else value.get("result") if "result" in value else value.get("content") | |
| out.append( | |
| { | |
| "call_id": call_id, | |
| "type": raw_type or role or "tool_result", | |
| "payload_text": json.dumps(payload, default=str, sort_keys=True), | |
| } | |
| ) | |
| for item in value.values(): | |
| collect_tool_results(item, out) | |
| def build_image(case: dict[str, Any], image: str) -> None: | |
| dockerfile = str( | |
| case.get("dockerfile") | |
| or dockerfile_for_target(str(case["target"]), str(case.get("version", image.rsplit(":", 1)[-1]))) | |
| ) | |
| with tempfile.TemporaryDirectory(prefix="vendor-repro-build-") as tmp: | |
| path = Path(tmp) / "Dockerfile" | |
| path.write_text(dockerfile, encoding="utf-8") | |
| cmd = [ | |
| "docker", | |
| "build", | |
| "--build-arg", | |
| f"RUNNER_UID={os.getuid()}", | |
| "--build-arg", | |
| f"RUNNER_GID={os.getgid()}", | |
| "-t", | |
| image, | |
| "-f", | |
| str(path), | |
| tmp, | |
| ] | |
| run_checked(cmd) | |
| def run_checked(cmd: list[str]) -> None: | |
| print("+ " + " ".join(cmd), flush=True) | |
| proc = subprocess.run(cmd, text=True) | |
| if proc.returncode != 0: | |
| raise SystemExit(proc.returncode) | |
| def derive_image(case: dict[str, Any]) -> str: | |
| if case.get("image"): | |
| return str(case["image"]) | |
| return f"vendor-repro-{case['target']}:{case.get('version', 'latest')}" | |
| def target_command(target: str, base_url: str, workspace: str, timeout_s: int) -> tuple[list[str], dict[str, str]]: | |
| if target == "claude": | |
| return ( | |
| [ | |
| "claude", | |
| "-p", | |
| PROMPT, | |
| "--dangerously-skip-permissions", | |
| "--output-format", | |
| "text", | |
| "--model", | |
| "claude-3-5-sonnet-20241022", | |
| ], | |
| { | |
| "ANTHROPIC_BASE_URL": base_url, | |
| "ANTHROPIC_API_KEY": "vendor-repro-fake-key", | |
| "ANTHROPIC_AUTH_TOKEN": "vendor-repro-fake-key", | |
| "DISABLE_AUTOUPDATER": "1", | |
| "CLAUDE_CODE_DISABLE_TELEMETRY": "1", | |
| "DISABLE_TELEMETRY": "1", | |
| "DISABLE_ERROR_REPORTING": "1", | |
| "DISABLE_NON_ESSENTIAL_MODEL_CALLS": "1", | |
| }, | |
| ) | |
| if target == "codex": | |
| base = base_url.rstrip("/") | |
| return ( | |
| [ | |
| "codex", | |
| "exec", | |
| "-c", | |
| 'model_provider="vendor-repro"', | |
| "-c", | |
| 'model_providers.vendor-repro.name="vendor-repro"', | |
| "-c", | |
| f'model_providers.vendor-repro.base_url="{base}/v1"', | |
| "-c", | |
| 'model_providers.vendor-repro.wire_api="responses"', | |
| "-c", | |
| "model_providers.vendor-repro.requires_openai_auth=false", | |
| "-c", | |
| 'approval_policy="never"', | |
| "-c", | |
| 'sandbox_mode="danger-full-access"', | |
| "-c", | |
| 'model="gpt-4"', | |
| "-c", | |
| 'model_reasoning_effort="low"', | |
| "--skip-git-repo-check", | |
| PROMPT, | |
| ], | |
| {"OPENAI_API_KEY": "vendor-repro-fake-key"}, | |
| ) | |
| if target == "every_code": | |
| base = base_url.rstrip("/") | |
| return ( | |
| [ | |
| "coder", | |
| "exec", | |
| "-c", | |
| 'model_provider="vendor-repro"', | |
| "-c", | |
| 'model_providers.vendor-repro.name="vendor-repro"', | |
| "-c", | |
| f'model_providers.vendor-repro.base_url="{base}/v1"', | |
| "-c", | |
| 'model_providers.vendor-repro.wire_api="responses"', | |
| "-c", | |
| "model_providers.vendor-repro.requires_openai_auth=false", | |
| "-c", | |
| 'approval_policy="never"', | |
| "-c", | |
| 'sandbox_mode="danger-full-access"', | |
| "-c", | |
| 'model="gpt-4"', | |
| "-c", | |
| 'model_reasoning_effort="low"', | |
| "--skip-git-repo-check", | |
| PROMPT, | |
| ], | |
| {"OPENAI_API_KEY": "vendor-repro-fake-key"}, | |
| ) | |
| if target == "gitagent": | |
| inner = " ".join( | |
| [ | |
| "git", | |
| "config", | |
| "--global", | |
| "--add", | |
| "safe.directory", | |
| shlex.quote(workspace), | |
| "&&", | |
| "exec", | |
| "gitagent", | |
| "--dir", | |
| shlex.quote(workspace), | |
| "--model", | |
| "openai:gpt-4", | |
| shlex.quote(PROMPT), | |
| ] | |
| ) | |
| return ( | |
| [ | |
| "bash", | |
| "-c", | |
| inner, | |
| ], | |
| { | |
| "OPENAI_API_KEY": "vendor-repro-fake-key", | |
| "GITAGENT_MODEL_BASE_URL": base_url.rstrip("/") + "/v1", | |
| "GIT_AUTHOR_NAME": "Vendor Repro", | |
| "GIT_AUTHOR_EMAIL": "[email protected]", | |
| "GIT_COMMITTER_NAME": "Vendor Repro", | |
| "GIT_COMMITTER_EMAIL": "[email protected]", | |
| }, | |
| ) | |
| if target == "open_codex": | |
| return ( | |
| [ | |
| "open-codex", | |
| "-q", | |
| "--provider", | |
| "ollama", | |
| "--approval-mode", | |
| "full-auto", | |
| "--model", | |
| "gpt-4", | |
| PROMPT, | |
| ], | |
| { | |
| "OLLAMA_BASE_URL": base_url.rstrip("/") + "/v1", | |
| "NO_UPDATE_NOTIFIER": "1", | |
| "CI": "1", | |
| }, | |
| ) | |
| if target == "zeroclaw": | |
| return ( | |
| [ | |
| "bash", | |
| "-c", | |
| "mkdir -p \"$ZEROCLAW_CONFIG_DIR\" && " | |
| "zeroclaw --config-dir \"$ZEROCLAW_CONFIG_DIR\" onboard --force " | |
| "--provider \"$ZEROCLAW_PROVIDER\" --api-key vendor-repro-fake-key " | |
| "--model \"$ZEROCLAW_MODEL\" --memory none " | |
| "> \"$ZEROCLAW_CONFIG_DIR/onboard.log\" 2>&1 && " | |
| "exec zeroclaw --config-dir \"$ZEROCLAW_CONFIG_DIR\" agent " | |
| "--provider \"$ZEROCLAW_PROVIDER\" --model \"$ZEROCLAW_MODEL\" " | |
| "-m \"$VENDOR_REPRO_PROMPT\"", | |
| ], | |
| { | |
| "ZEROCLAW_CONFIG_DIR": "/workspace/vendor_repro_zeroclaw_home", | |
| "ZEROCLAW_PROVIDER": "custom:" + base_url.rstrip("/") + "/v1", | |
| "ZEROCLAW_MODEL": "gpt-4", | |
| "VENDOR_REPRO_PROMPT": PROMPT, | |
| }, | |
| ) | |
| if target == "goose": | |
| return ( | |
| [ | |
| "goose", | |
| "run", | |
| "--no-profile", | |
| "--with-builtin", | |
| "developer", | |
| "--max-turns", | |
| str(max(3, timeout_s // 10)), | |
| "--output-format", | |
| "json", | |
| "--text", | |
| PROMPT, | |
| ], | |
| { | |
| "ANTHROPIC_API_KEY": "vendor-repro-fake-key", | |
| "ANTHROPIC_HOST": base_url.rstrip(), | |
| "GOOSE_PROVIDER": "anthropic", | |
| "GOOSE_MODEL": "claude-sonnet-4-5", | |
| }, | |
| ) | |
| if target == "crush": | |
| return ( | |
| [ | |
| "crush", | |
| "--cwd", | |
| workspace, | |
| "--data-dir", | |
| workspace + "/vendor_repro_crush_home/.local/share/crush", | |
| "run", | |
| "--quiet", | |
| "-m", | |
| "local-repro/gpt-4", | |
| PROMPT, | |
| ], | |
| {"OPENAI_API_KEY": "vendor-repro-fake-key"}, | |
| ) | |
| if target == "qwen": | |
| return ( | |
| [ | |
| "/home/runner/.npm-global/bin/qwen", | |
| "--chat-recording=false", | |
| "--auth-type", | |
| "openai", | |
| "--openai-api-key", | |
| "vendor-repro-fake-key", | |
| "--openai-base-url", | |
| base_url.rstrip("/") + "/v1", | |
| "--model", | |
| "gpt-4", | |
| "--approval-mode", | |
| "yolo", | |
| "--output-format", | |
| "text", | |
| "-p", | |
| PROMPT, | |
| ], | |
| {"OPENAI_API_KEY": "vendor-repro-fake-key", "QWEN_CODE_SUPPRESS_YOLO_WARNING": "1"}, | |
| ) | |
| if target == "gemini": | |
| base = base_url.rstrip("/") | |
| return ( | |
| [ | |
| "gemini", | |
| "-p", | |
| PROMPT, | |
| "--model", | |
| "gemini-3-flash", | |
| "--output-format", | |
| "text", | |
| "--skip-trust", | |
| "--yolo", | |
| ], | |
| { | |
| "GEMINI_API_KEY": "vendor-repro-fake-key", | |
| "GOOGLE_GEMINI_BASE_URL": base, | |
| "GEMINI_TELEMETRY_ENABLED": "false", | |
| "NO_COLOR": "1", | |
| "CI": "1", | |
| }, | |
| ) | |
| if target == "forgecode": | |
| base = base_url.rstrip("/") + "/v1" | |
| data_home = "/workspace/vendor_repro_forgecode_home" | |
| return ( | |
| [ | |
| "bash", | |
| "-c", | |
| "mkdir -p \"$HOME/.forge\" && " | |
| "printf '[{\"id\":\"openai_compatible\",\"auth_details\":{\"api_key\":\"%s\"},\"url_params\":{\"OPENAI_URL\":\"%s\"}}]\\n' " | |
| "\"$OPENAI_API_KEY\" \"$OPENAI_URL\" > \"$HOME/.forge/.credentials.json\" && " | |
| "forge config set model openai_compatible \"$MODELLOCK_MODEL\" " | |
| "> \"$HOME/forge-config.log\" 2>&1 && " | |
| "exec forge -C /workspace -p \"$MODELLOCK_PROMPT\"", | |
| ], | |
| { | |
| "HOME": data_home, | |
| "XDG_CONFIG_HOME": data_home + "/.config", | |
| "XDG_DATA_HOME": data_home + "/.local/share", | |
| "XDG_CACHE_HOME": data_home + "/.cache", | |
| "OPENAI_API_KEY": "vendor-repro-fake-key", | |
| "OPENAI_URL": base, | |
| "MODELLOCK_MODEL": "gpt-4o", | |
| "MODELLOCK_PROMPT": PROMPT, | |
| }, | |
| ) | |
| if target == "groq_code_cli": | |
| data_home = "/workspace/vendor_repro_groq_code_cli_home" | |
| return ( | |
| [ | |
| "bash", | |
| "-c", | |
| """cd /workspace && exec node --input-type=module <<'VENDOR_REPRO_GROQ_CODE_JS' | |
| import { pathToFileURL } from "node:url"; | |
| const modulePath = process.env.VENDOR_REPRO_GROQ_CODE_AGENT_MODULE | |
| || "/home/runner/.npm-global/lib/node_modules/groq-code-cli/dist/core/agent.js"; | |
| const { Agent } = await import(pathToFileURL(modulePath).href); | |
| const agent = await Agent.create( | |
| process.env.VENDOR_REPRO_MODEL || "moonshotai/kimi-k2-instruct", | |
| 1.0, | |
| null, | |
| false, | |
| ); | |
| agent.setToolCallbacks({ | |
| onFinalMessage: (message) => console.log(message || ""), | |
| onToolStart: (name, args) => console.error("VENDOR_REPRO_TOOL_START " + name + " " + JSON.stringify(args)), | |
| onToolEnd: (name, result) => console.error("VENDOR_REPRO_TOOL_END " + name + " " + JSON.stringify(result).slice(0, 4000)), | |
| onToolApproval: async () => ({ approved: true, autoApproveSession: true }), | |
| onThinkingText: () => {}, | |
| onMaxIterations: async () => false, | |
| onApiUsage: () => {}, | |
| }); | |
| await agent.chat(process.env.VENDOR_REPRO_PROMPT || ""); | |
| VENDOR_REPRO_GROQ_CODE_JS""", | |
| ], | |
| { | |
| "HOME": data_home, | |
| "XDG_CONFIG_HOME": data_home + "/.config", | |
| "XDG_DATA_HOME": data_home + "/.local/share", | |
| "XDG_CACHE_HOME": data_home + "/.cache", | |
| "GROQ_API_KEY": "vendor-repro-fake-key", | |
| "GROQ_BASE_URL": base_url.rstrip("/"), | |
| "VENDOR_REPRO_MODEL": "moonshotai/kimi-k2-instruct", | |
| "VENDOR_REPRO_PROMPT": PROMPT, | |
| }, | |
| ) | |
| if target == "letta_code": | |
| base = base_url.rstrip("/") + "/v1" | |
| data_home = "/workspace/vendor_repro_letta_code_home" | |
| return ( | |
| [ | |
| "bash", | |
| "-c", | |
| "mkdir -p \"$LETTA_LOCAL_BACKEND_DIR\" && " | |
| "letta --backend local connect lmstudio " | |
| "--base-url \"$MODELLOCK_LETTA_BASE_URL\" --timeout 5s " | |
| "> \"$HOME/letta-connect.log\" 2>&1 && " | |
| "exec letta --backend local --new-agent " | |
| "--model \"$MODELLOCK_LETTA_MODEL\" " | |
| "-p \"$MODELLOCK_LETTA_PROMPT\" " | |
| "--output-format text --no-skills --no-mods " | |
| "--no-system-info-reminder --no-memfs " | |
| "--personality blank --base-tools \"\"", | |
| ], | |
| { | |
| "HOME": data_home, | |
| "XDG_CONFIG_HOME": data_home + "/.config", | |
| "XDG_DATA_HOME": data_home + "/.local/share", | |
| "XDG_CACHE_HOME": data_home + "/.cache", | |
| "LETTA_LOCAL_BACKEND_DIR": data_home + "/lc-local-backend", | |
| "LETTA_SKIP_KEYCHAIN_CHECK": "1", | |
| "LETTA_DISABLE_SESSION_PERSIST": "1", | |
| "LETTA_LOCAL_BACKEND_NO_MEMFS": "1", | |
| "NO_COLOR": "1", | |
| "CI": "1", | |
| "TERM": "dumb", | |
| "MODELLOCK_LETTA_BASE_URL": base, | |
| "MODELLOCK_LETTA_MODEL": "lmstudio/gpt-4", | |
| "MODELLOCK_LETTA_PROMPT": PROMPT, | |
| }, | |
| ) | |
| if target == "kilo_code": | |
| return ( | |
| [ | |
| "kilo", | |
| "run", | |
| "--model", | |
| "vendorrepro/gpt-4", | |
| "--auto", | |
| "--format", | |
| "json", | |
| "--dir", | |
| workspace, | |
| PROMPT, | |
| ], | |
| { | |
| "HOME": "/workspace/vendor_repro_kilo_code_home", | |
| "XDG_CONFIG_HOME": "/workspace/vendor_repro_kilo_code_home/.config", | |
| "XDG_DATA_HOME": "/workspace/vendor_repro_kilo_code_home/.local/share", | |
| "XDG_CACHE_HOME": "/workspace/vendor_repro_kilo_code_home/.cache", | |
| "NO_COLOR": "1", | |
| "CI": "1", | |
| "TERM": "dumb", | |
| }, | |
| ) | |
| if target == "mini_kode": | |
| return ( | |
| [ | |
| "mini-kode", | |
| PROMPT, | |
| "--approval-mode", | |
| "yolo", | |
| "--work-dir", | |
| workspace, | |
| ], | |
| { | |
| "HOME": "/workspace/vendor_repro_mini_kode_home", | |
| "XDG_CONFIG_HOME": "/workspace/vendor_repro_mini_kode_home/.config", | |
| "XDG_DATA_HOME": "/workspace/vendor_repro_mini_kode_home/.local/share", | |
| "XDG_CACHE_HOME": "/workspace/vendor_repro_mini_kode_home/.cache", | |
| "MINIKODE_API_KEY": "vendor-repro-fake-key", | |
| "MINIKODE_BASE_URL": base_url.rstrip("/") + "/v1", | |
| "MINIKODE_MODEL": "gpt-4", | |
| "OPENAI_API_KEY": "vendor-repro-fake-key", | |
| }, | |
| ) | |
| if target == "pi": | |
| return ( | |
| [ | |
| "pi", | |
| "-p", | |
| "--provider", | |
| "vendorrepro", | |
| "--model", | |
| "gpt-4", | |
| "--approve", | |
| "--no-session", | |
| "--no-context-files", | |
| "--no-extensions", | |
| "--no-skills", | |
| "--no-prompt-templates", | |
| "--no-themes", | |
| "--tools", | |
| "read,bash,edit,write", | |
| "--api-key", | |
| "vendor-repro-fake-key", | |
| PROMPT, | |
| ], | |
| { | |
| "HOME": "/workspace/vendor_repro_pi_home", | |
| "XDG_CONFIG_HOME": "/workspace/vendor_repro_pi_home/.config", | |
| "XDG_DATA_HOME": "/workspace/vendor_repro_pi_home/.local/share", | |
| "XDG_CACHE_HOME": "/workspace/vendor_repro_pi_home/.cache", | |
| "PI_CODING_AGENT_DIR": "/workspace/vendor_repro_pi_home", | |
| "PI_OFFLINE": "1", | |
| }, | |
| ) | |
| if target == "kode_cli": | |
| return ( | |
| [ | |
| "kode", | |
| "-p", | |
| PROMPT, | |
| "--cwd", | |
| workspace, | |
| "--permission-mode", | |
| "bypassPermissions", | |
| "--dangerously-skip-permissions", | |
| "--no-session-persistence", | |
| "--tools", | |
| "Bash,Read,Edit,Write", | |
| "--output-format", | |
| "text", | |
| "--model", | |
| "gpt-4", | |
| ], | |
| { | |
| "HOME": "/workspace/vendor_repro_kode_cli_home", | |
| "XDG_CONFIG_HOME": "/workspace/vendor_repro_kode_cli_home/.config", | |
| "XDG_DATA_HOME": "/workspace/vendor_repro_kode_cli_home/.local/share", | |
| "XDG_CACHE_HOME": "/workspace/vendor_repro_kode_cli_home/.cache", | |
| "KODE_CONFIG_DIR": "/workspace/vendor_repro_kode_cli_home/config", | |
| "KODE_OFFLINE": "1", | |
| "OPENAI_API_KEY": "vendor-repro-fake-key", | |
| }, | |
| ) | |
| if target == "free_code": | |
| return ( | |
| [ | |
| "free-code", | |
| "-p", | |
| PROMPT, | |
| "--bare", | |
| "--dangerously-skip-permissions", | |
| "--output-format", | |
| "text", | |
| "--max-turns", | |
| str(max(3, timeout_s // 20)), | |
| "--model", | |
| "claude-3-5-sonnet-20241022", | |
| "--no-session-persistence", | |
| ], | |
| { | |
| "HOME": "/workspace/vendor_repro_free_code_home", | |
| "XDG_CONFIG_HOME": "/workspace/vendor_repro_free_code_home/.config", | |
| "XDG_DATA_HOME": "/workspace/vendor_repro_free_code_home/.local/share", | |
| "XDG_CACHE_HOME": "/workspace/vendor_repro_free_code_home/.cache", | |
| "ANTHROPIC_BASE_URL": base_url.rstrip("/"), | |
| "ANTHROPIC_API_KEY": "vendor-repro-fake-key", | |
| "ANTHROPIC_AUTH_TOKEN": "vendor-repro-fake-key", | |
| "API_TIMEOUT_MS": "30000", | |
| "DISABLE_AUTOUPDATER": "1", | |
| "CLAUDE_CODE_DISABLE_AUTOUPDATER": "1", | |
| "CLAUDE_CODE_DISABLE_TELEMETRY": "1", | |
| "DISABLE_TELEMETRY": "1", | |
| "DISABLE_ERROR_REPORTING": "1", | |
| "DISABLE_NON_ESSENTIAL_MODEL_CALLS": "1", | |
| "CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC": "1", | |
| "CLAUDE_CODE_DISABLE_CLAUDE_MDS": "1", | |
| "NO_COLOR": "1", | |
| "CI": "1", | |
| "TERM": "dumb", | |
| }, | |
| ) | |
| if target == "qqcode": | |
| return ( | |
| [ | |
| "qqcode", | |
| "--prompt", | |
| PROMPT, | |
| "--auto-approve", | |
| "--max-turns", | |
| "3", | |
| "--output", | |
| "text", | |
| ], | |
| { | |
| "HOME": "/workspace/vendor_repro_qqcode_home", | |
| "XDG_CONFIG_HOME": "/workspace/vendor_repro_qqcode_home/.config", | |
| "XDG_DATA_HOME": "/workspace/vendor_repro_qqcode_home/.local/share", | |
| "XDG_CACHE_HOME": "/workspace/vendor_repro_qqcode_home/.cache", | |
| "QQCODE_HOME": "/workspace/vendor_repro_qqcode_home", | |
| "OPENAI_API_KEY": "vendor-repro-fake-key", | |
| "NO_COLOR": "1", | |
| }, | |
| ) | |
| if target == "crab_code": | |
| base = base_url.rstrip("/") + "/v1" | |
| return ( | |
| [ | |
| "crab", | |
| "--provider", | |
| "openai", | |
| "--model", | |
| "gpt-4o-mini", | |
| "--config-dir", | |
| "/workspace/vendor_repro_crab_code_home", | |
| "--dangerously-skip-permissions", | |
| "--bare", | |
| "--no-session-persistence", | |
| "--max-turns", | |
| "1", | |
| "--print", | |
| PROMPT, | |
| ], | |
| { | |
| "HOME": "/workspace/vendor_repro_crab_code_home", | |
| "XDG_CONFIG_HOME": "/workspace/vendor_repro_crab_code_home/.config", | |
| "XDG_DATA_HOME": "/workspace/vendor_repro_crab_code_home/.local/share", | |
| "XDG_CACHE_HOME": "/workspace/vendor_repro_crab_code_home/.cache", | |
| "CRAB_CONFIG_DIR": "/workspace/vendor_repro_crab_code_home", | |
| "CRAB_API_PROVIDER": "openai", | |
| "CRAB_API_KEY": "vendor-repro-fake-key", | |
| "CRAB_MODEL": "gpt-4o-mini", | |
| "CRAB_BASE_URL": base, | |
| "OPENAI_API_KEY": "vendor-repro-fake-key", | |
| "OPENAI_BASE_URL": base, | |
| "NO_COLOR": "1", | |
| }, | |
| ) | |
| if target == "neovate_code": | |
| base = base_url.rstrip("/") + "/v1" | |
| return ( | |
| [ | |
| "neovate", | |
| "-q", | |
| "--model", | |
| "openai/gpt-4.1", | |
| "--output-format", | |
| "json", | |
| "--cwd", | |
| "/workspace", | |
| "--tools", | |
| "{\"write\":true,\"bash\":true}", | |
| PROMPT, | |
| ], | |
| { | |
| "HOME": "/workspace/vendor_repro_neovate_code_home", | |
| "XDG_CONFIG_HOME": "/workspace/vendor_repro_neovate_code_home/.config", | |
| "XDG_DATA_HOME": "/workspace/vendor_repro_neovate_code_home/.local/share", | |
| "XDG_CACHE_HOME": "/workspace/vendor_repro_neovate_code_home/.cache", | |
| "OPENAI_API_KEY": "vendor-repro-fake-key", | |
| "OPENAI_API_BASE": base, | |
| "NEOVATE_SELF_UPDATE": "none", | |
| "NO_COLOR": "1", | |
| "CI": "1", | |
| "TERM": "dumb", | |
| }, | |
| ) | |
| if target == "mini_swe_agent": | |
| base = base_url.rstrip("/") + "/v1" | |
| output_path = "/workspace/vendor_repro_mini_swe_agent_home/trajectory.json" | |
| return ( | |
| [ | |
| "mini", | |
| "--model", | |
| "openai/gpt-4o", | |
| "--task", | |
| PROMPT, | |
| "--yolo", | |
| "--exit-immediately", | |
| "--cost-limit", | |
| "0", | |
| "--output", | |
| output_path, | |
| ], | |
| { | |
| "HOME": "/workspace/vendor_repro_mini_swe_agent_home", | |
| "XDG_CONFIG_HOME": "/workspace/vendor_repro_mini_swe_agent_home/.config", | |
| "XDG_DATA_HOME": "/workspace/vendor_repro_mini_swe_agent_home/.local/share", | |
| "XDG_CACHE_HOME": "/workspace/vendor_repro_mini_swe_agent_home/.cache", | |
| "OPENAI_API_KEY": "vendor-repro-fake-key", | |
| "OPENAI_API_BASE": base, | |
| "MSWEA_CONFIGURED": "true", | |
| "MSWEA_COST_TRACKING": "ignore_errors", | |
| "MSWEA_SILENT_STARTUP": "1", | |
| "NO_COLOR": "1", | |
| "CI": "1", | |
| "TERM": "dumb", | |
| }, | |
| ) | |
| if target == "claurst": | |
| return ( | |
| [ | |
| "claurst", | |
| "--provider", | |
| "openai", | |
| "--api-base", | |
| base_url.rstrip("/") + "/v1", | |
| "--api-key", | |
| "vendor-repro-fake-key", | |
| "--model", | |
| "gpt-4o-mini", | |
| "--print", | |
| PROMPT, | |
| "--max-turns", | |
| "1", | |
| "--output-format", | |
| "text", | |
| "--dangerously-skip-permissions", | |
| ], | |
| { | |
| "HOME": "/workspace/vendor_repro_claurst_home", | |
| "XDG_CONFIG_HOME": "/workspace/vendor_repro_claurst_home/.config", | |
| "XDG_DATA_HOME": "/workspace/vendor_repro_claurst_home/.local/share", | |
| "XDG_CACHE_HOME": "/workspace/vendor_repro_claurst_home/.cache", | |
| "CLAURST_DISABLE_NONESSENTIAL_TRAFFIC": "1", | |
| "CLAURST_SKIP_PROMPT_HISTORY": "1", | |
| "NO_COLOR": "1", | |
| }, | |
| ) | |
| if target == "picocode": | |
| return ( | |
| [ | |
| "picocode", | |
| "--provider", | |
| "openai", | |
| "--model", | |
| "gpt-4o-mini", | |
| "--yolo", | |
| "true", | |
| "--tool-call-limit", | |
| "8", | |
| "--quiet", | |
| PROMPT, | |
| ], | |
| { | |
| "HOME": "/workspace/vendor_repro_picocode_home", | |
| "XDG_CONFIG_HOME": "/workspace/vendor_repro_picocode_home/.config", | |
| "XDG_DATA_HOME": "/workspace/vendor_repro_picocode_home/.local/share", | |
| "XDG_CACHE_HOME": "/workspace/vendor_repro_picocode_home/.cache", | |
| "OPENAI_API_KEY": "vendor-repro-fake-key", | |
| "OPENAI_BASE_URL": base_url.rstrip("/") + "/v1", | |
| "NO_COLOR": "1", | |
| }, | |
| ) | |
| if target == "hermes_agent": | |
| return ( | |
| [ | |
| "hermes", | |
| "--ignore-rules", | |
| "--yolo", | |
| "--provider", | |
| "anthropic", | |
| "--model", | |
| "claude-3-5-sonnet-20241022", | |
| "-z", | |
| PROMPT, | |
| ], | |
| { | |
| "HOME": "/workspace/vendor_repro_hermes_agent_home", | |
| "XDG_CONFIG_HOME": "/workspace/vendor_repro_hermes_agent_home/.config", | |
| "XDG_DATA_HOME": "/workspace/vendor_repro_hermes_agent_home/.local/share", | |
| "XDG_CACHE_HOME": "/workspace/vendor_repro_hermes_agent_home/.cache", | |
| "HERMES_ALLOW_PRIVATE_URLS": "1", | |
| "OPENAI_API_KEY": "vendor-repro-fake-key", | |
| "OPENAI_BASE_URL": base_url.rstrip("/") + "/v1", | |
| "ANTHROPIC_API_KEY": "vendor-repro-fake-key", | |
| "ANTHROPIC_BASE_URL": base_url.rstrip("/"), | |
| "NO_COLOR": "1", | |
| "CI": "1", | |
| }, | |
| ) | |
| if target == "cline": | |
| base = base_url.rstrip("/") + "/v1" | |
| data_dir = "/workspace/vendor_repro_cline_data" | |
| model = "gpt-4" | |
| auth_cmd = " ".join( | |
| [ | |
| shlex.quote("cline"), | |
| "auth", | |
| "--provider", | |
| "openai-native", | |
| "--apikey", | |
| "vendor-repro-fake-key", | |
| "--modelid", | |
| shlex.quote(model), | |
| "--baseurl", | |
| shlex.quote(base), | |
| "--data-dir", | |
| shlex.quote(data_dir), | |
| ] | |
| ) | |
| run_cmd = " ".join( | |
| [ | |
| "exec", | |
| shlex.quote("cline"), | |
| "--data-dir", | |
| shlex.quote(data_dir), | |
| "--cwd", | |
| shlex.quote(workspace), | |
| "-P", | |
| "openai-native", | |
| "-m", | |
| shlex.quote(model), | |
| "-k", | |
| "vendor-repro-fake-key", | |
| "--json", | |
| "--retries", | |
| "100", | |
| "-y", | |
| shlex.quote(PROMPT), | |
| ] | |
| ) | |
| return ( | |
| ["bash", "-c", auth_cmd + " && " + run_cmd], | |
| { | |
| "CLINE_DATA_DIR": data_dir, | |
| "CLINE_LOG_ENABLED": "0", | |
| "OPENAI_API_KEY": "vendor-repro-fake-key", | |
| }, | |
| ) | |
| if target == "opencode": | |
| return ( | |
| ["opencode", "run", "-m", "anthropic/claude-haiku-4-5-20251001", PROMPT], | |
| { | |
| "ANTHROPIC_API_KEY": "vendor-repro-fake-key", | |
| "ANTHROPIC_BASE_URL": base_url.rstrip("/") + "/v1", | |
| "OPENCODE_DISABLE_DEFAULT_PLUGINS": "1", | |
| }, | |
| ) | |
| if target == "openharness": | |
| return ( | |
| [ | |
| "oh", | |
| "run", | |
| PROMPT, | |
| "--model", | |
| "openai/gpt-4", | |
| "--trust", | |
| "--bare", | |
| "--no-session-persistence", | |
| "--output-format", | |
| "text", | |
| "--max-turns", | |
| "1", | |
| ], | |
| {"OPENAI_API_KEY": "vendor-repro-fake-key"}, | |
| ) | |
| if target == "continue": | |
| return ( | |
| [ | |
| "cn", | |
| "--config", | |
| "/workspace/vendor_repro_continue_home/config.yaml", | |
| "--auto", | |
| "--silent", | |
| "-p", | |
| PROMPT, | |
| ], | |
| { | |
| "CONTINUE_HOME": "/workspace/vendor_repro_continue_home", | |
| "OPENAI_API_KEY": "vendor-repro-fake-key", | |
| }, | |
| ) | |
| raise ValueError(f"unsupported target: {target}") | |
| def _write_qqcode_repro_config(path: Path, base_url: str, model: str = "gpt-4o-mini") -> None: | |
| active_model = f"vendorrepro/{model}" | |
| path.parent.mkdir(parents=True, exist_ok=True) | |
| path.write_text( | |
| f"active_model = {json.dumps(active_model)}\n" | |
| "api_timeout = 30\n" | |
| "\n" | |
| "[[providers]]\n" | |
| "name = \"vendorrepro\"\n" | |
| f"api_base = {json.dumps(base_url.rstrip('/') + '/v1')}\n" | |
| "api_key_env_var = \"OPENAI_API_KEY\"\n" | |
| "backend = \"generic\"\n" | |
| "\n" | |
| "[[models]]\n" | |
| f"name = {json.dumps(model)}\n" | |
| "provider = \"vendorrepro\"\n" | |
| f"alias = {json.dumps(active_model)}\n", | |
| encoding="utf-8", | |
| ) | |
| def _write_kilo_repro_config(path: Path, base_url: str) -> None: | |
| path.parent.mkdir(parents=True, exist_ok=True) | |
| payload = { | |
| "$schema": "https://app.kilo.ai/config.json", | |
| "provider": { | |
| "vendorrepro": { | |
| "npm": "@ai-sdk/openai-compatible", | |
| "name": "Vendor Repro Mock", | |
| "options": { | |
| "baseURL": base_url, | |
| "apiKey": "vendor-repro-fake-key", | |
| "timeout": 10000, | |
| "headerTimeout": 5000, | |
| "chunkTimeout": 5000, | |
| }, | |
| "models": { | |
| "gpt-4": { | |
| "name": "gpt-4", | |
| "tool_call": True, | |
| "temperature": True, | |
| "limit": {"context": 8192, "output": 1024}, | |
| "cost": {"input": 0, "output": 0}, | |
| } | |
| }, | |
| } | |
| }, | |
| "model": "vendorrepro/gpt-4", | |
| "permission": {"bash": "allow", "edit": "allow", "read": "allow", "list": "allow", "grep": "allow", "write": "allow"}, | |
| } | |
| path.write_text(json.dumps(payload, sort_keys=True), encoding="utf-8") | |
| def _write_pi_repro_models_config(path: Path, base_url: str) -> None: | |
| path.parent.mkdir(parents=True, exist_ok=True) | |
| payload = { | |
| "providers": { | |
| "vendorrepro": { | |
| "baseUrl": base_url, | |
| "api": "openai-completions", | |
| "apiKey": "vendor-repro-fake-key", | |
| "models": [ | |
| { | |
| "id": "gpt-4", | |
| "reasoning": False, | |
| "input": ["text"], | |
| "contextWindow": 128000, | |
| "maxTokens": 4096, | |
| "cost": {"input": 0, "output": 0, "cacheRead": 0, "cacheWrite": 0}, | |
| "compat": {"supportsDeveloperRole": False, "supportsReasoningEffort": False}, | |
| } | |
| ], | |
| } | |
| } | |
| } | |
| path.write_text(json.dumps(payload, sort_keys=True), encoding="utf-8") | |
| def _write_kode_repro_config(path: Path, base_url: str) -> None: | |
| path.parent.mkdir(parents=True, exist_ok=True) | |
| payload = { | |
| "theme": "dark", | |
| "hasCompletedOnboarding": True, | |
| "lastOnboardingVersion": "2.2.1", | |
| "primaryProvider": "custom-openai", | |
| "modelProfiles": [ | |
| { | |
| "name": "vendorrepro", | |
| "provider": "custom-openai", | |
| "modelName": "gpt-4", | |
| "baseURL": base_url, | |
| "apiKey": "vendor-repro-fake-key", | |
| "maxTokens": 4096, | |
| "contextLength": 128000, | |
| "isActive": True, | |
| "createdAt": 1760000000000, | |
| } | |
| ], | |
| "modelPointers": {"main": "gpt-4", "task": "gpt-4", "quick": "gpt-4", "compact": "gpt-4"}, | |
| } | |
| path.write_text(json.dumps(payload, sort_keys=True), encoding="utf-8") | |
| def prepare_workspace(root: Path, target: str, base_url: str, case: dict[str, Any]) -> dict[str, str]: | |
| root.mkdir(parents=True, exist_ok=True) | |
| (root / "README.md").write_text("# Vendor repro workspace\n", encoding="utf-8") | |
| files = case.get("workspace_files", {}) | |
| if isinstance(files, dict): | |
| for raw_path, content in files.items(): | |
| rel = Path(str(raw_path)) | |
| if rel.is_absolute() or ".." in rel.parts: | |
| raise ValueError(f"unsafe workspace path: {raw_path}") | |
| destination = root / rel | |
| destination.parent.mkdir(parents=True, exist_ok=True) | |
| destination.write_text(str(content), encoding="utf-8") | |
| env: dict[str, str] = {} | |
| if target == "crush": | |
| host_home = root / "vendor_repro_crush_home" | |
| host_config_dir = host_home / ".config" / "crush" | |
| host_data_dir = host_home / ".local" / "share" / "crush" | |
| host_config_dir.mkdir(parents=True, exist_ok=True) | |
| host_data_dir.mkdir(parents=True, exist_ok=True) | |
| (host_config_dir / "crush.json").write_text( | |
| json.dumps( | |
| { | |
| "$schema": "https://charm.land/crush.json", | |
| "providers": { | |
| "local-repro": { | |
| "type": "openai-compat", | |
| "base_url": base_url.rstrip("/") + "/v1", | |
| "api_key": "vendor-repro-fake-key", | |
| "models": [ | |
| { | |
| "id": "gpt-4", | |
| "name": "Local repro gpt-4", | |
| "context_window": 128000, | |
| "default_max_tokens": 2000, | |
| } | |
| ], | |
| } | |
| }, | |
| }, | |
| sort_keys=True, | |
| ), | |
| encoding="utf-8", | |
| ) | |
| env.update( | |
| { | |
| "HOME": "/workspace/vendor_repro_crush_home", | |
| "XDG_CONFIG_HOME": "/workspace/vendor_repro_crush_home/.config", | |
| "XDG_DATA_HOME": "/workspace/vendor_repro_crush_home/.local/share", | |
| "CRUSH_GLOBAL_CONFIG": "/workspace/vendor_repro_crush_home/.config/crush", | |
| "CRUSH_GLOBAL_DATA": "/workspace/vendor_repro_crush_home/.local/share/crush", | |
| } | |
| ) | |
| if target == "goose": | |
| home = root / "vendor_repro_goose_home" | |
| for rel in (".config", ".cache", ".local/share", ".local/state"): | |
| (home / rel).mkdir(parents=True, exist_ok=True) | |
| env.update( | |
| { | |
| "HOME": "/workspace/vendor_repro_goose_home", | |
| "XDG_CONFIG_HOME": "/workspace/vendor_repro_goose_home/.config", | |
| "XDG_CACHE_HOME": "/workspace/vendor_repro_goose_home/.cache", | |
| "XDG_DATA_HOME": "/workspace/vendor_repro_goose_home/.local/share", | |
| "XDG_STATE_HOME": "/workspace/vendor_repro_goose_home/.local/state", | |
| } | |
| ) | |
| if target == "qwen": | |
| host_home = root / "vendor_repro_qwen_home" | |
| host_config = host_home / ".qwen" | |
| host_config.mkdir(parents=True, exist_ok=True) | |
| (host_config / "settings.json").write_text( | |
| json.dumps( | |
| { | |
| "modelProviders": { | |
| "openai": [ | |
| { | |
| "id": "gpt-4", | |
| "name": "gpt-4", | |
| "baseUrl": base_url.rstrip("/") + "/v1", | |
| "envKey": "OPENAI_API_KEY", | |
| } | |
| ] | |
| }, | |
| "env": {"OPENAI_API_KEY": "vendor-repro-fake-key"}, | |
| "security": {"auth": {"selectedType": "openai"}}, | |
| "model": {"name": "gpt-4"}, | |
| "telemetry": {"enabled": False}, | |
| }, | |
| sort_keys=True, | |
| ), | |
| encoding="utf-8", | |
| ) | |
| env["HOME"] = "/workspace/vendor_repro_qwen_home" | |
| if target == "gemini": | |
| host_home = root / "vendor_repro_gemini_home" | |
| host_config = host_home / ".gemini" | |
| host_config.mkdir(parents=True, exist_ok=True) | |
| (host_config / "settings.json").write_text( | |
| json.dumps( | |
| { | |
| "security": {"auth": {"selectedType": "gateway", "useExternal": True}}, | |
| "telemetry": {"enabled": False}, | |
| }, | |
| sort_keys=True, | |
| ), | |
| encoding="utf-8", | |
| ) | |
| env["HOME"] = "/workspace/vendor_repro_gemini_home" | |
| if target == "forgecode": | |
| host_home = root / "vendor_repro_forgecode_home" | |
| for rel in (".config", ".cache", ".local/share", ".forge"): | |
| (host_home / rel).mkdir(parents=True, exist_ok=True) | |
| env.update( | |
| { | |
| "HOME": "/workspace/vendor_repro_forgecode_home", | |
| "XDG_CONFIG_HOME": "/workspace/vendor_repro_forgecode_home/.config", | |
| "XDG_CACHE_HOME": "/workspace/vendor_repro_forgecode_home/.cache", | |
| "XDG_DATA_HOME": "/workspace/vendor_repro_forgecode_home/.local/share", | |
| } | |
| ) | |
| if target == "groq_code_cli": | |
| host_home = root / "vendor_repro_groq_code_cli_home" | |
| for rel in (".config", ".cache", ".local/share"): | |
| (host_home / rel).mkdir(parents=True, exist_ok=True) | |
| env.update( | |
| { | |
| "HOME": "/workspace/vendor_repro_groq_code_cli_home", | |
| "XDG_CONFIG_HOME": "/workspace/vendor_repro_groq_code_cli_home/.config", | |
| "XDG_CACHE_HOME": "/workspace/vendor_repro_groq_code_cli_home/.cache", | |
| "XDG_DATA_HOME": "/workspace/vendor_repro_groq_code_cli_home/.local/share", | |
| } | |
| ) | |
| if target == "letta_code": | |
| host_home = root / "vendor_repro_letta_code_home" | |
| for rel in (".config", ".cache", ".local/share", "lc-local-backend"): | |
| (host_home / rel).mkdir(parents=True, exist_ok=True) | |
| env.update( | |
| { | |
| "HOME": "/workspace/vendor_repro_letta_code_home", | |
| "XDG_CONFIG_HOME": "/workspace/vendor_repro_letta_code_home/.config", | |
| "XDG_CACHE_HOME": "/workspace/vendor_repro_letta_code_home/.cache", | |
| "XDG_DATA_HOME": "/workspace/vendor_repro_letta_code_home/.local/share", | |
| "LETTA_LOCAL_BACKEND_DIR": "/workspace/vendor_repro_letta_code_home/lc-local-backend", | |
| } | |
| ) | |
| if target == "kilo_code": | |
| host_home = root / "vendor_repro_kilo_code_home" | |
| for rel in (".config/kilo", ".cache", ".local/share", ".local/state"): | |
| (host_home / rel).mkdir(parents=True, exist_ok=True) | |
| _write_kilo_repro_config(host_home / ".config" / "kilo" / "kilo.jsonc", base_url.rstrip("/") + "/v1") | |
| env.update( | |
| { | |
| "HOME": "/workspace/vendor_repro_kilo_code_home", | |
| "XDG_CONFIG_HOME": "/workspace/vendor_repro_kilo_code_home/.config", | |
| "XDG_CACHE_HOME": "/workspace/vendor_repro_kilo_code_home/.cache", | |
| "XDG_DATA_HOME": "/workspace/vendor_repro_kilo_code_home/.local/share", | |
| "XDG_STATE_HOME": "/workspace/vendor_repro_kilo_code_home/.local/state", | |
| } | |
| ) | |
| if target == "mini_kode": | |
| host_home = root / "vendor_repro_mini_kode_home" | |
| for rel in (".config", ".cache", ".local/share"): | |
| (host_home / rel).mkdir(parents=True, exist_ok=True) | |
| env.update( | |
| { | |
| "HOME": "/workspace/vendor_repro_mini_kode_home", | |
| "XDG_CONFIG_HOME": "/workspace/vendor_repro_mini_kode_home/.config", | |
| "XDG_CACHE_HOME": "/workspace/vendor_repro_mini_kode_home/.cache", | |
| "XDG_DATA_HOME": "/workspace/vendor_repro_mini_kode_home/.local/share", | |
| } | |
| ) | |
| if target == "pi": | |
| host_home = root / "vendor_repro_pi_home" | |
| for rel in (".config", ".cache", ".local/share"): | |
| (host_home / rel).mkdir(parents=True, exist_ok=True) | |
| _write_pi_repro_models_config(host_home / "models.json", base_url.rstrip("/") + "/v1") | |
| env.update( | |
| { | |
| "HOME": "/workspace/vendor_repro_pi_home", | |
| "XDG_CONFIG_HOME": "/workspace/vendor_repro_pi_home/.config", | |
| "XDG_CACHE_HOME": "/workspace/vendor_repro_pi_home/.cache", | |
| "XDG_DATA_HOME": "/workspace/vendor_repro_pi_home/.local/share", | |
| "PI_CODING_AGENT_DIR": "/workspace/vendor_repro_pi_home", | |
| } | |
| ) | |
| if target == "kode_cli": | |
| host_home = root / "vendor_repro_kode_cli_home" | |
| for rel in (".config", ".cache", ".local/share", "config"): | |
| (host_home / rel).mkdir(parents=True, exist_ok=True) | |
| _write_kode_repro_config(host_home / "config" / "config.json", base_url.rstrip("/") + "/v1") | |
| env.update( | |
| { | |
| "HOME": "/workspace/vendor_repro_kode_cli_home", | |
| "XDG_CONFIG_HOME": "/workspace/vendor_repro_kode_cli_home/.config", | |
| "XDG_CACHE_HOME": "/workspace/vendor_repro_kode_cli_home/.cache", | |
| "XDG_DATA_HOME": "/workspace/vendor_repro_kode_cli_home/.local/share", | |
| "KODE_CONFIG_DIR": "/workspace/vendor_repro_kode_cli_home/config", | |
| } | |
| ) | |
| if target == "cline": | |
| host_data = root / "vendor_repro_cline_data" | |
| host_data.mkdir(parents=True, exist_ok=True) | |
| host_data.chmod(0o777) | |
| env["CLINE_DATA_DIR"] = "/workspace/vendor_repro_cline_data" | |
| if target == "codex" and case.get("mcp_slow_tools_list"): | |
| host_home = root / "vendor_repro_codex_home" | |
| probe = root / "vendor_repro_mcp_probe" | |
| host_home.mkdir(parents=True, exist_ok=True) | |
| probe.mkdir(parents=True, exist_ok=True) | |
| server_path = probe / "server.py" | |
| log_path = "/workspace/vendor_repro_mcp_probe/events.jsonl" | |
| delay_ms = int(case.get("mcp_tools_list_delay_ms", 2500)) | |
| server_path.write_text(_mcp_slow_tools_list_server(log_path, delay_ms), encoding="utf-8") | |
| server_path.chmod(0o755) | |
| project = "/workspace".replace("\\", "\\\\").replace('"', '\\"') | |
| config = [ | |
| f'[projects."{project}"]', | |
| 'trust_level = "trusted"', | |
| "", | |
| "[mcp_servers.vendorreproslow]", | |
| 'command = "python3"', | |
| 'args = ["/workspace/vendor_repro_mcp_probe/server.py"]', | |
| "", | |
| "[mcp_servers.vendorreproslow.env]", | |
| f"VENDOR_REPRO_MCP_LOG = {json.dumps(log_path)}", | |
| f"VENDOR_REPRO_MCP_TOOLS_LIST_DELAY_MS = {json.dumps(str(delay_ms))}", | |
| ] | |
| (host_home / "config.toml").write_text("\n".join(config) + "\n", encoding="utf-8") | |
| env.update({"CODEX_HOME": "/workspace/vendor_repro_codex_home", "HOME": "/workspace/vendor_repro_codex_home"}) | |
| if target == "continue": | |
| host_home = root / "vendor_repro_continue_home" | |
| host_home.mkdir(parents=True, exist_ok=True) | |
| config_path = host_home / "config.yaml" | |
| config_path.write_text( | |
| "name: Vendor Reproducer\n" | |
| "version: 1.0.0\n" | |
| "schema: v1\n" | |
| "models:\n" | |
| " - name: Vendor GPT-4\n" | |
| " provider: openai\n" | |
| " model: gpt-4\n" | |
| " apiKey: vendor-repro-fake-key\n" | |
| " apiBase: " + base_url.rstrip("/") + "/v1\n", | |
| encoding="utf-8", | |
| ) | |
| config_path.chmod(0o666) | |
| env["CONTINUE_HOME"] = "/workspace/vendor_repro_continue_home" | |
| if target == "openharness": | |
| host_home = root / "vendor_repro_openharness_home" | |
| for rel in (".config", ".cache", ".local/share", ".local/state"): | |
| (host_home / rel).mkdir(parents=True, exist_ok=True) | |
| oh_dir = root / ".oh" | |
| oh_dir.mkdir(parents=True, exist_ok=True) | |
| (oh_dir / "config.yaml").write_text( | |
| "provider: openai\n" | |
| "model: gpt-4\n" | |
| f"baseUrl: {base_url.rstrip('/')}/v1\n" | |
| "permissionMode: trust\n", | |
| encoding="utf-8", | |
| ) | |
| env.update( | |
| { | |
| "HOME": "/workspace/vendor_repro_openharness_home", | |
| "XDG_CONFIG_HOME": "/workspace/vendor_repro_openharness_home/.config", | |
| "XDG_CACHE_HOME": "/workspace/vendor_repro_openharness_home/.cache", | |
| "XDG_DATA_HOME": "/workspace/vendor_repro_openharness_home/.local/share", | |
| "XDG_STATE_HOME": "/workspace/vendor_repro_openharness_home/.local/state", | |
| } | |
| ) | |
| if target == "opencode": | |
| host_home = root / "vendor_repro_opencode_home" | |
| for rel in (".config", ".cache", ".local/share", ".local/share/opencode"): | |
| (host_home / rel).mkdir(parents=True, exist_ok=True) | |
| env.update( | |
| { | |
| "HOME": "/workspace/vendor_repro_opencode_home", | |
| "XDG_CONFIG_HOME": "/workspace/vendor_repro_opencode_home/.config", | |
| "XDG_CACHE_HOME": "/workspace/vendor_repro_opencode_home/.cache", | |
| "XDG_DATA_HOME": "/workspace/vendor_repro_opencode_home/.local/share", | |
| "OPENCODE_DISABLE_DEFAULT_PLUGINS": "1", | |
| } | |
| ) | |
| if target == "gitagent": | |
| host_home = root / "vendor_repro_gitagent_home" | |
| for rel in (".config", ".cache", ".local/share"): | |
| (host_home / rel).mkdir(parents=True, exist_ok=True) | |
| subprocess.run(["git", "init"], cwd=root, check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) | |
| subprocess.run(["git", "config", "user.email", "[email protected]"], cwd=root, check=True) | |
| subprocess.run(["git", "config", "user.name", "Vendor Repro"], cwd=root, check=True) | |
| subprocess.run(["git", "add", "."], cwd=root, check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) | |
| subprocess.run( | |
| ["git", "commit", "-m", "initial vendor repro workspace", "--allow-empty"], | |
| cwd=root, | |
| check=True, | |
| stdout=subprocess.DEVNULL, | |
| stderr=subprocess.DEVNULL, | |
| ) | |
| env.update( | |
| { | |
| "HOME": "/workspace/vendor_repro_gitagent_home", | |
| "XDG_CONFIG_HOME": "/workspace/vendor_repro_gitagent_home/.config", | |
| "XDG_CACHE_HOME": "/workspace/vendor_repro_gitagent_home/.cache", | |
| "XDG_DATA_HOME": "/workspace/vendor_repro_gitagent_home/.local/share", | |
| } | |
| ) | |
| if target == "open_codex": | |
| host_home = root / "vendor_repro_open_codex_home" | |
| for rel in (".config", ".cache", ".local/share"): | |
| (host_home / rel).mkdir(parents=True, exist_ok=True) | |
| env.update( | |
| { | |
| "HOME": "/workspace/vendor_repro_open_codex_home", | |
| "XDG_CONFIG_HOME": "/workspace/vendor_repro_open_codex_home/.config", | |
| "XDG_CACHE_HOME": "/workspace/vendor_repro_open_codex_home/.cache", | |
| "XDG_DATA_HOME": "/workspace/vendor_repro_open_codex_home/.local/share", | |
| } | |
| ) | |
| if target == "zeroclaw": | |
| host_home = root / "vendor_repro_zeroclaw_home" | |
| for rel in (".config", ".cache", ".local/share"): | |
| (host_home / rel).mkdir(parents=True, exist_ok=True) | |
| env.update( | |
| { | |
| "HOME": "/workspace/vendor_repro_zeroclaw_home", | |
| "XDG_CONFIG_HOME": "/workspace/vendor_repro_zeroclaw_home/.config", | |
| "XDG_CACHE_HOME": "/workspace/vendor_repro_zeroclaw_home/.cache", | |
| "XDG_DATA_HOME": "/workspace/vendor_repro_zeroclaw_home/.local/share", | |
| "ZEROCLAW_CONFIG_DIR": "/workspace/vendor_repro_zeroclaw_home", | |
| } | |
| ) | |
| if target == "free_code": | |
| host_home = root / "vendor_repro_free_code_home" | |
| for rel in (".config", ".cache", ".local/share", ".local/state"): | |
| (host_home / rel).mkdir(parents=True, exist_ok=True) | |
| env.update( | |
| { | |
| "HOME": "/workspace/vendor_repro_free_code_home", | |
| "XDG_CONFIG_HOME": "/workspace/vendor_repro_free_code_home/.config", | |
| "XDG_CACHE_HOME": "/workspace/vendor_repro_free_code_home/.cache", | |
| "XDG_DATA_HOME": "/workspace/vendor_repro_free_code_home/.local/share", | |
| "XDG_STATE_HOME": "/workspace/vendor_repro_free_code_home/.local/state", | |
| } | |
| ) | |
| if target == "qqcode": | |
| host_home = root / "vendor_repro_qqcode_home" | |
| for rel in (".config", ".cache", ".local/share"): | |
| (host_home / rel).mkdir(parents=True, exist_ok=True) | |
| _write_qqcode_repro_config(host_home / "config.toml", base_url) | |
| env.update( | |
| { | |
| "HOME": "/workspace/vendor_repro_qqcode_home", | |
| "XDG_CONFIG_HOME": "/workspace/vendor_repro_qqcode_home/.config", | |
| "XDG_CACHE_HOME": "/workspace/vendor_repro_qqcode_home/.cache", | |
| "XDG_DATA_HOME": "/workspace/vendor_repro_qqcode_home/.local/share", | |
| "QQCODE_HOME": "/workspace/vendor_repro_qqcode_home", | |
| } | |
| ) | |
| if target == "crab_code": | |
| host_home = root / "vendor_repro_crab_code_home" | |
| for rel in (".config", ".cache", ".local/share"): | |
| (host_home / rel).mkdir(parents=True, exist_ok=True) | |
| env.update( | |
| { | |
| "HOME": "/workspace/vendor_repro_crab_code_home", | |
| "XDG_CONFIG_HOME": "/workspace/vendor_repro_crab_code_home/.config", | |
| "XDG_CACHE_HOME": "/workspace/vendor_repro_crab_code_home/.cache", | |
| "XDG_DATA_HOME": "/workspace/vendor_repro_crab_code_home/.local/share", | |
| } | |
| ) | |
| if target == "neovate_code": | |
| host_home = root / "vendor_repro_neovate_code_home" | |
| for rel in (".config", ".cache", ".local/share"): | |
| (host_home / rel).mkdir(parents=True, exist_ok=True) | |
| env.update( | |
| { | |
| "HOME": "/workspace/vendor_repro_neovate_code_home", | |
| "XDG_CONFIG_HOME": "/workspace/vendor_repro_neovate_code_home/.config", | |
| "XDG_CACHE_HOME": "/workspace/vendor_repro_neovate_code_home/.cache", | |
| "XDG_DATA_HOME": "/workspace/vendor_repro_neovate_code_home/.local/share", | |
| } | |
| ) | |
| if target == "mini_swe_agent": | |
| host_home = root / "vendor_repro_mini_swe_agent_home" | |
| for rel in (".config", ".cache", ".local/share"): | |
| (host_home / rel).mkdir(parents=True, exist_ok=True) | |
| env.update( | |
| { | |
| "HOME": "/workspace/vendor_repro_mini_swe_agent_home", | |
| "XDG_CONFIG_HOME": "/workspace/vendor_repro_mini_swe_agent_home/.config", | |
| "XDG_CACHE_HOME": "/workspace/vendor_repro_mini_swe_agent_home/.cache", | |
| "XDG_DATA_HOME": "/workspace/vendor_repro_mini_swe_agent_home/.local/share", | |
| } | |
| ) | |
| if target == "claurst": | |
| host_home = root / "vendor_repro_claurst_home" | |
| for rel in (".config", ".cache", ".local/share"): | |
| (host_home / rel).mkdir(parents=True, exist_ok=True) | |
| env.update( | |
| { | |
| "HOME": "/workspace/vendor_repro_claurst_home", | |
| "XDG_CONFIG_HOME": "/workspace/vendor_repro_claurst_home/.config", | |
| "XDG_CACHE_HOME": "/workspace/vendor_repro_claurst_home/.cache", | |
| "XDG_DATA_HOME": "/workspace/vendor_repro_claurst_home/.local/share", | |
| } | |
| ) | |
| if target == "picocode": | |
| host_home = root / "vendor_repro_picocode_home" | |
| for rel in (".config", ".cache", ".local/share"): | |
| (host_home / rel).mkdir(parents=True, exist_ok=True) | |
| env.update( | |
| { | |
| "HOME": "/workspace/vendor_repro_picocode_home", | |
| "XDG_CONFIG_HOME": "/workspace/vendor_repro_picocode_home/.config", | |
| "XDG_CACHE_HOME": "/workspace/vendor_repro_picocode_home/.cache", | |
| "XDG_DATA_HOME": "/workspace/vendor_repro_picocode_home/.local/share", | |
| } | |
| ) | |
| if target == "hermes_agent": | |
| host_home = root / "vendor_repro_hermes_agent_home" | |
| for rel in (".config", ".cache", ".local/share"): | |
| (host_home / rel).mkdir(parents=True, exist_ok=True) | |
| env.update( | |
| { | |
| "HOME": "/workspace/vendor_repro_hermes_agent_home", | |
| "XDG_CONFIG_HOME": "/workspace/vendor_repro_hermes_agent_home/.config", | |
| "XDG_CACHE_HOME": "/workspace/vendor_repro_hermes_agent_home/.cache", | |
| "XDG_DATA_HOME": "/workspace/vendor_repro_hermes_agent_home/.local/share", | |
| } | |
| ) | |
| if case.get("side_effect_probe"): | |
| env["REPRO_SIDE_EFFECT_ROOT"] = "/side-effects" | |
| return env | |
| def _mcp_slow_tools_list_server(log_path: str, delay_ms: int) -> str: | |
| return f"""#!/usr/bin/env python3 | |
| import json | |
| import os | |
| import sys | |
| import time | |
| LOG_PATH = os.environ.get("VENDOR_REPRO_MCP_LOG") or {log_path!r} | |
| DELAY_MS = int(os.environ.get("VENDOR_REPRO_MCP_TOOLS_LIST_DELAY_MS", {str(delay_ms)!r})) | |
| def log(event, **attrs): | |
| os.makedirs(os.path.dirname(LOG_PATH), exist_ok=True) | |
| with open(LOG_PATH, "a", encoding="utf-8") as handle: | |
| handle.write(json.dumps({{"ts": time.time(), "event": event, **attrs}}, sort_keys=True) + "\\n") | |
| handle.flush() | |
| def send(obj): | |
| sys.stdout.write(json.dumps(obj) + "\\n") | |
| sys.stdout.flush() | |
| log("server_send", method=obj.get("method"), id=obj.get("id"), has_error="error" in obj) | |
| def read_msg(): | |
| line = sys.stdin.buffer.readline() | |
| if not line: | |
| return None | |
| if line.lstrip().startswith(b"{{"): | |
| return json.loads(line.decode("utf-8")) | |
| headers = [] | |
| while line not in (b"\\r\\n", b"\\n", b""): | |
| headers.append(line.decode("ascii", "replace").strip()) | |
| line = sys.stdin.buffer.readline() | |
| length = 0 | |
| for header in headers: | |
| if header.lower().startswith("content-length:"): | |
| length = int(header.split(":", 1)[1].strip()) | |
| if length <= 0: | |
| return None | |
| return json.loads(sys.stdin.buffer.read(length).decode("utf-8")) | |
| log("server_start") | |
| while True: | |
| req = read_msg() | |
| if req is None: | |
| log("stdin_eof") | |
| break | |
| method = req.get("method") | |
| log("client_request", method=method, id=req.get("id")) | |
| if method == "initialize": | |
| send({{"jsonrpc":"2.0","id":req.get("id"),"result":{{"protocolVersion":"2025-06-18","capabilities":{{"tools":{{"listChanged":False}}}},"serverInfo":{{"name":"vendor-repro-slow","version":"1.0.0"}}}}}}) | |
| elif method == "notifications/initialized": | |
| continue | |
| elif method == "tools/list": | |
| time.sleep(DELAY_MS / 1000.0) | |
| send({{"jsonrpc":"2.0","id":req.get("id"),"result":{{"tools":[{{"name":"ping","description":"slow MCP ping","inputSchema":{{"type":"object","properties":{{}},"additionalProperties":False}}}}]}}}}) | |
| elif method == "tools/call": | |
| send({{"jsonrpc":"2.0","id":req.get("id"),"result":{{"content":[{{"type":"text","text":"pong"}}]}}}}) | |
| else: | |
| send({{"jsonrpc":"2.0","id":req.get("id"),"error":{{"code":-32601,"message":"method not found"}}}}) | |
| """ | |
| def run_target( | |
| case: dict[str, Any], | |
| image: str, | |
| base_url: str, | |
| workspace: Path, | |
| side_effects: Path, | |
| extra_env: dict[str, str], | |
| ) -> subprocess.CompletedProcess[bytes]: | |
| target = str(case["target"]) | |
| timeout_s = int(case.get("timeout_s", 90)) | |
| inner, env = target_command(target, base_url, "/workspace", timeout_s) | |
| env = {**extra_env, **env} | |
| cidfile = workspace / ".vendor_repro_container_id" | |
| try: | |
| cidfile.unlink() | |
| except FileNotFoundError: | |
| pass | |
| cmd = [ | |
| "docker", | |
| "run", | |
| "--rm", | |
| "--cidfile", | |
| str(cidfile), | |
| "-v", | |
| f"{workspace}:/workspace:rw", | |
| "-v", | |
| f"{side_effects}:/side-effects:rw", | |
| "-w", | |
| "/workspace", | |
| *docker_network_args(base_url), | |
| "--cpus", | |
| str(case.get("docker_cpus", DEFAULT_DOCKER_CPUS)), | |
| "--memory", | |
| str(case.get("docker_memory", DEFAULT_DOCKER_MEMORY)), | |
| "-e", | |
| f"PATH={DOCKER_PATH}", | |
| ] | |
| for key, value in sorted(env.items()): | |
| cmd.extend(["-e", f"{key}={value}"]) | |
| docker_user = os.environ.get("MODELLOCK_REPRO_DOCKER_USER") | |
| if docker_user is None and target in {"claude", "forgecode", "claurst", "free_code"}: | |
| docker_user = f"{os.getuid()}:{os.getgid()}" | |
| cmd.extend(["--entrypoint", "/usr/bin/env"]) | |
| if docker_user: | |
| cmd.extend(["--user", docker_user]) | |
| cmd.append(image) | |
| if case.get("side_effect_probe"): | |
| pid_file = str(case.get("pid_file", "child.pid")) | |
| alive_file = str(case.get("alive_file", "child.alive")) | |
| quoted_inner = " ".join(shlex.quote(part) for part in inner) | |
| probe = ( | |
| "status=0; " | |
| + quoted_inner | |
| + " || status=$?; " | |
| + "python3 - <<'PY'\n" | |
| + "import os, pathlib\n" | |
| + "root = pathlib.Path('/side-effects')\n" | |
| + f"pid_path = root / {pid_file!r}\n" | |
| + f"alive_path = root / {alive_file!r}\n" | |
| + "alive = False\n" | |
| + "pid_text = ''\n" | |
| + "if pid_path.exists():\n" | |
| + " pid_text = pid_path.read_text().strip()\n" | |
| + " try:\n" | |
| + " os.kill(int(pid_text), 0)\n" | |
| + " alive = True\n" | |
| + " except Exception:\n" | |
| + " alive = False\n" | |
| + "alive_path.write_text(('alive' if alive else 'dead') + '\\n' + pid_text)\n" | |
| + "PY\n" | |
| + "exit $status" | |
| ) | |
| cmd.extend(["bash", "-c", probe]) | |
| else: | |
| cmd.extend(inner) | |
| print("+ " + " ".join(cmd), flush=True) | |
| proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, start_new_session=True) | |
| signal_after = case.get("signal_after_ms") | |
| if signal_after is not None: | |
| def send_signal() -> None: | |
| time.sleep(float(signal_after) / 1000.0) | |
| try: | |
| os.killpg(os.getpgid(proc.pid), signal.SIGINT) | |
| except ProcessLookupError: | |
| pass | |
| threading.Thread(target=send_signal, daemon=True).start() | |
| try: | |
| stdout, stderr = proc.communicate(timeout=timeout_s) | |
| result = subprocess.CompletedProcess(cmd, proc.returncode, stdout, stderr) | |
| except subprocess.TimeoutExpired: | |
| try: | |
| os.killpg(os.getpgid(proc.pid), signal.SIGTERM) | |
| except ProcessLookupError: | |
| pass | |
| cleanup_docker_container(cidfile) | |
| try: | |
| stdout, stderr = proc.communicate(timeout=5) | |
| except subprocess.TimeoutExpired: | |
| try: | |
| os.killpg(os.getpgid(proc.pid), signal.SIGKILL) | |
| except ProcessLookupError: | |
| pass | |
| cleanup_docker_container(cidfile) | |
| stdout, stderr = proc.communicate() | |
| result = subprocess.CompletedProcess(cmd, proc.returncode, stdout, stderr) | |
| result.timed_out = True # type: ignore[attr-defined] | |
| return result | |
| result.timed_out = False # type: ignore[attr-defined] | |
| return result | |
| def provider_host_for_container() -> str: | |
| override = os.environ.get("MODELLOCK_REPRO_PROVIDER_HOST") | |
| if override: | |
| return override | |
| if sys.platform.startswith("linux"): | |
| return "127.0.0.1" | |
| return "host.docker.internal" | |
| def docker_network_args(base_url: str) -> list[str]: | |
| if "127.0.0.1" in base_url or "localhost" in base_url: | |
| return ["--network", "host"] | |
| if "host.docker.internal" in base_url: | |
| return ["--add-host", "host.docker.internal:host-gateway"] | |
| return [] | |
| def cleanup_docker_container(cidfile: Path) -> None: | |
| try: | |
| container_id = cidfile.read_text(encoding="utf-8").strip() | |
| except FileNotFoundError: | |
| return | |
| if not container_id: | |
| return | |
| try: | |
| subprocess.run( | |
| ["docker", "rm", "-f", container_id], | |
| stdout=subprocess.DEVNULL, | |
| stderr=subprocess.DEVNULL, | |
| timeout=10, | |
| check=False, | |
| ) | |
| except (OSError, subprocess.TimeoutExpired): | |
| pass | |
| def temporary_workspace() -> Any: | |
| base = Path(os.environ.get("MODELLOCK_REPRO_TMPDIR", Path.cwd())).resolve() | |
| base.mkdir(parents=True, exist_ok=True) | |
| return _BestEffortTemporaryDirectory(prefix="vendor-repro-workspace-", dir=str(base)) | |
| class _BestEffortTemporaryDirectory: | |
| def __init__(self, prefix: str, dir: str) -> None: | |
| self.name = tempfile.mkdtemp(prefix=prefix, dir=dir) | |
| def __enter__(self) -> str: | |
| return self.name | |
| def __exit__(self, exc_type: object, exc: object, tb: object) -> None: | |
| try: | |
| shutil.rmtree(self.name) | |
| except Exception: | |
| pass | |
| def expanded_case_runs(case: dict[str, Any]) -> list[dict[str, Any]]: | |
| runs = case.get("runs") | |
| if not isinstance(runs, list): | |
| return [case] | |
| base = {key: value for key, value in case.items() if key != "runs"} | |
| expanded: list[dict[str, Any]] = [] | |
| for index, run in enumerate(runs, 1): | |
| if not isinstance(run, dict): | |
| raise ValueError(f"invalid run entry: {run!r}") | |
| merged = {**base, **run} | |
| merged.setdefault("run_name", f"run{index}") | |
| expanded.append(merged) | |
| return expanded | |
| def is_pid_alive(pid: int) -> bool: | |
| if pid <= 0: | |
| return False | |
| try: | |
| os.kill(pid, 0) | |
| except ProcessLookupError: | |
| return False | |
| except PermissionError: | |
| return True | |
| return True | |
| def response_events(requests: list[dict[str, Any]]) -> list[dict[str, Any]]: | |
| events: list[dict[str, Any]] = [] | |
| for request in requests: | |
| for event in request.get("response_events", []): | |
| if isinstance(event, dict): | |
| events.append(event) | |
| return events | |
| def max_tool_result_count_in_one_request(requests: list[dict[str, Any]], call_id: str = "") -> int: | |
| max_count = 0 | |
| for request in requests: | |
| results: list[dict[str, Any]] = [] | |
| collect_tool_results(request.get("body"), results) | |
| counts: dict[str, int] = {} | |
| for result in results: | |
| current = str(result.get("call_id", "")) | |
| if call_id and current != call_id: | |
| continue | |
| counts[current] = counts.get(current, 0) + 1 | |
| if counts: | |
| max_count = max(max_count, max(counts.values())) | |
| return max_count | |
| def evaluate( | |
| case: dict[str, Any], | |
| requests: list[dict[str, Any]], | |
| proc: subprocess.CompletedProcess[bytes], | |
| side_effects: Path, | |
| ) -> tuple[bool, list[str]]: | |
| results: list[dict[str, Any]] = [] | |
| for request in requests: | |
| collect_tool_results(request.get("body"), results) | |
| result_ids = [str(result["call_id"]) for result in results] | |
| counts: dict[str, int] = {} | |
| for call_id in result_ids: | |
| counts[call_id] = counts.get(call_id, 0) + 1 | |
| events = response_events(requests) | |
| late_events = [ | |
| event | |
| for event in events | |
| if event.get("after_stop") and not event.get("broken_pipe") and ("delta" in str(event.get("raw", ""))) | |
| ] | |
| total_result_bytes = sum(len(str(result.get("payload_text", "")).encode("utf-8", errors="ignore")) for result in results) | |
| closed_mid_stream = sum(1 for request in requests if request.get("closed_mid_stream")) | |
| timed_out = bool(getattr(proc, "timed_out", False)) | |
| checks = case.get("checks", []) | |
| messages = [ | |
| f"process_exit={proc.returncode}", | |
| f"process_timed_out={timed_out}", | |
| f"provider_requests={len(requests)}", | |
| f"provider_paths={sorted({str(request.get('path', '')) for request in requests})}", | |
| f"tool_result_counts={dict(sorted(counts.items()))}", | |
| f"tool_result_bytes={total_result_bytes}", | |
| f"closed_mid_stream={closed_mid_stream}", | |
| f"late_stream_events_after_stop={len(late_events)}", | |
| ] | |
| ok = True | |
| for check in checks: | |
| kind = check.get("kind") | |
| call_id = str(check.get("call_id", "")) | |
| if kind == "duplicate_tool_result": | |
| minimum = int(check.get("min_count", 2)) | |
| actual = counts.get(call_id, 0) if call_id else (max(counts.values()) if counts else 0) | |
| passed = actual >= minimum | |
| messages.append(f"duplicate_tool_result {call_id or '<any>'}: observed={actual} expected>={minimum}") | |
| ok = ok and passed | |
| elif kind == "duplicate_tool_result_in_request": | |
| minimum = int(check.get("min_count", 2)) | |
| actual = max_tool_result_count_in_one_request(requests, call_id) | |
| passed = actual >= minimum | |
| messages.append(f"duplicate_tool_result_in_request {call_id or '<any>'}: observed={actual} expected>={minimum}") | |
| ok = ok and passed | |
| elif kind == "tool_result": | |
| actual = counts.get(call_id, 0) | |
| passed = actual >= int(check.get("min_count", 1)) | |
| messages.append(f"tool_result {call_id}: observed={actual} expected>={int(check.get('min_count', 1))}") | |
| ok = ok and passed | |
| elif kind == "tool_result_payload_contains": | |
| needle = str(check.get("text", "")) | |
| actual = any(result["call_id"] == call_id and needle in result.get("payload_text", "") for result in results) | |
| messages.append(f"tool_result_payload_contains {call_id} {needle!r}: observed={actual}") | |
| ok = ok and actual | |
| elif kind == "any_tool_result_after_signal": | |
| minimum = int(check.get("min_count", 1)) | |
| actual = len(result_ids) | |
| passed = actual >= minimum | |
| messages.append(f"tool_result_after_signal: observed={actual} expected>={minimum}") | |
| ok = ok and passed | |
| elif kind == "late_stream_events_after_stop": | |
| minimum = int(check.get("min_count", 1)) | |
| actual = len(late_events) | |
| passed = actual >= minimum | |
| messages.append(f"late_stream_events_after_stop: observed={actual} expected>={minimum}") | |
| ok = ok and passed | |
| elif kind == "pid_file_alive": | |
| raw_path = str(check.get("path", "")) | |
| rel = Path(raw_path) | |
| if rel.is_absolute() or ".." in rel.parts: | |
| raise ValueError(f"unsafe side-effect path: {raw_path}") | |
| pid_file = side_effects / rel | |
| pid_text = pid_file.read_text(encoding="utf-8").strip() if pid_file.exists() else "" | |
| alive_file = side_effects / str(check.get("alive_file", case.get("alive_file", "child.alive"))) | |
| if alive_file.exists(): | |
| alive_text = alive_file.read_text(encoding="utf-8").strip().splitlines() | |
| actual = bool(alive_text and alive_text[0] == "alive") | |
| if len(alive_text) > 1 and not pid_text: | |
| pid_text = alive_text[1] | |
| else: | |
| try: | |
| pid = int(pid_text) | |
| except ValueError: | |
| pid = -1 | |
| actual = is_pid_alive(pid) | |
| messages.append(f"pid_file_alive {raw_path}: pid={pid_text or '<missing>'} observed={actual}") | |
| ok = ok and actual | |
| elif kind == "side_effect_file_exists": | |
| raw_path = str(check.get("path", "")) | |
| rel = Path(raw_path) | |
| if rel.is_absolute() or ".." in rel.parts: | |
| raise ValueError(f"unsafe side-effect path: {raw_path}") | |
| path = side_effects / rel | |
| minimum = int(check.get("min_bytes", 1)) | |
| actual_bytes = path.stat().st_size if path.exists() else 0 | |
| passed = actual_bytes >= minimum | |
| messages.append(f"side_effect_file_exists {raw_path}: bytes={actual_bytes} expected>={minimum}") | |
| ok = ok and passed | |
| elif kind == "side_effect_file_contains": | |
| raw_path = str(check.get("path", "")) | |
| rel = Path(raw_path) | |
| if rel.is_absolute() or ".." in rel.parts: | |
| raise ValueError(f"unsafe side-effect path: {raw_path}") | |
| needle = str(check.get("text", "")) | |
| path = side_effects / rel | |
| content = path.read_text(encoding="utf-8", errors="replace") if path.exists() else "" | |
| actual = needle in content | |
| messages.append(f"side_effect_file_contains {raw_path} {needle!r}: observed={actual}") | |
| ok = ok and actual | |
| elif kind == "side_effect_file_occurrences": | |
| raw_path = str(check.get("path", "")) | |
| rel = Path(raw_path) | |
| if rel.is_absolute() or ".." in rel.parts: | |
| raise ValueError(f"unsafe side-effect path: {raw_path}") | |
| needle = str(check.get("text", "")) | |
| minimum = int(check.get("min_count", 1)) | |
| path = side_effects / rel | |
| content = path.read_text(encoding="utf-8", errors="replace") if path.exists() else "" | |
| actual = content.count(needle) if needle else 0 | |
| passed = actual >= minimum | |
| messages.append(f"side_effect_file_occurrences {raw_path} {needle!r}: observed={actual} expected>={minimum}") | |
| ok = ok and passed | |
| elif kind == "provider_requests_at_least": | |
| minimum = int(check.get("min_count", 1)) | |
| actual = len(requests) | |
| passed = actual >= minimum | |
| messages.append(f"provider_requests_at_least: observed={actual} expected>={minimum}") | |
| ok = ok and passed | |
| elif kind == "first_provider_request_delay_ms_at_least": | |
| minimum = int(check.get("min_ms", 1)) | |
| if requests: | |
| first_delay_ms = int((float(requests[0].get("ts", 0)) - float(case.get("_provider_started_ts", 0))) * 1000) | |
| else: | |
| first_delay_ms = -1 | |
| passed = first_delay_ms >= minimum | |
| messages.append(f"first_provider_request_delay_ms_at_least: observed={first_delay_ms} expected>={minimum}") | |
| ok = ok and passed | |
| elif kind == "tool_result_bytes_at_least": | |
| minimum = int(check.get("min_bytes", 1)) | |
| passed = total_result_bytes >= minimum | |
| messages.append(f"tool_result_bytes_at_least: observed={total_result_bytes} expected>={minimum}") | |
| ok = ok and passed | |
| elif kind == "large_tool_results_at_least": | |
| threshold = int(check.get("min_bytes", 200_000)) | |
| minimum = int(check.get("min_count", 1)) | |
| actual = sum( | |
| 1 | |
| for result in results | |
| if len(str(result.get("payload_text", "")).encode("utf-8", errors="ignore")) >= threshold | |
| ) | |
| passed = actual >= minimum | |
| messages.append(f"large_tool_results_at_least: observed={actual} expected>={minimum}") | |
| ok = ok and passed | |
| elif kind == "closed_mid_stream_at_least": | |
| minimum = int(check.get("min_count", 1)) | |
| passed = closed_mid_stream >= minimum | |
| messages.append(f"closed_mid_stream_at_least: observed={closed_mid_stream} expected>={minimum}") | |
| ok = ok and passed | |
| elif kind == "nonzero_or_timeout": | |
| passed = bool(proc.returncode not in (0, None) or timed_out) | |
| messages.append(f"nonzero_or_timeout: returncode={proc.returncode} timed_out={timed_out}") | |
| ok = ok and passed | |
| elif kind == "timeout_or_signal": | |
| passed = bool(timed_out or (proc.returncode is not None and proc.returncode < 0) or proc.returncode == 124) | |
| messages.append(f"timeout_or_signal: returncode={proc.returncode} timed_out={timed_out}") | |
| ok = ok and passed | |
| else: | |
| ok = False | |
| messages.append(f"unknown check: {kind}") | |
| return ok, messages | |
| def main() -> int: | |
| parser = argparse.ArgumentParser() | |
| parser.add_argument("case", type=Path, nargs="?") | |
| parser.add_argument("--skip-build", action="store_true") | |
| args = parser.parse_args() | |
| if args.case is None: | |
| if EMBEDDED_CASE is None: | |
| parser.error("case JSON is required unless this file embeds a case") | |
| case = EMBEDDED_CASE | |
| else: | |
| case = json.loads(args.case.read_text(encoding="utf-8")) | |
| runs = expanded_case_runs(case) | |
| if not args.skip_build: | |
| built: set[str] = set() | |
| for run in runs: | |
| image = derive_image(run) | |
| if image not in built: | |
| build_image(run, image) | |
| built.add(image) | |
| all_ok = True | |
| for run in runs: | |
| image = derive_image(run) | |
| state = ProviderState(run) | |
| server = ProviderServer(("127.0.0.1", 0), state) | |
| run["_provider_started_ts"] = time.time() | |
| thread = threading.Thread(target=server.serve_forever, daemon=True) | |
| thread.start() | |
| _host, port = server.server_address | |
| base_url = f"http://{provider_host_for_container()}:{port}" | |
| try: | |
| with temporary_workspace() as tmp: | |
| workspace = Path(tmp) | |
| side_effects = workspace / "side-effects" | |
| side_effects.mkdir(parents=True, exist_ok=True) | |
| extra_env = prepare_workspace(workspace, str(run["target"]), base_url, run) | |
| proc = run_target(run, image, base_url, workspace, side_effects, extra_env) | |
| requests = state.snapshot_requests() | |
| ok, messages = evaluate(run, requests, proc, side_effects) | |
| all_ok = all_ok and ok | |
| print("\n=== Reproduction summary ===") | |
| print(f"case={case.get('slug', '<embedded>')} run={run.get('run_name', run['target'])}") | |
| print(f"target={run['target']} image={image}") | |
| print(f"process_exit={proc.returncode}") | |
| for message in messages: | |
| print(message) | |
| if proc.stdout: | |
| print("stdout_tail=" + proc.stdout[-1000:].decode("utf-8", errors="replace")) | |
| if proc.stderr: | |
| print("stderr_tail=" + proc.stderr[-1000:].decode("utf-8", errors="replace")) | |
| print("RUN_REPRODUCED" if ok else "RUN_NOT_REPRODUCED") | |
| finally: | |
| server.shutdown() | |
| server.server_close() | |
| thread.join(timeout=2) | |
| print("REPRODUCED" if all_ok else "NOT_REPRODUCED") | |
| return 0 if all_ok else 1 | |
| if __name__ == "__main__": | |
| raise SystemExit(main()) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment