Skip to content

Commit a13cda7

Browse files
andre-mottaclaude
andcommitted
Disable agent-eval-harness for Claude runs in the OpenShell E2E
The plugin's SessionStart hook bulk-installs MLflow with uv. uv reuses a policy-DNS synthetic IP past its ~30s mapping lifetime, and the resulting transparent_tcp_mapping_denied while other relays are open crashes the OpenShell supervisor (upstream NVIDIA/OpenShell#3396). Enable every other shipped plugin via AGENT_ENABLED_PLUGINS until the supervisor survives that denial. Co-Authored-By: Claude Opus 5.5 <[email protected]> Signed-off-by: Andre Lustosa <[email protected]>
1 parent efa3f40 commit a13cda7

1 file changed

Lines changed: 21 additions & 0 deletions

File tree

‎tests/e2e/e2e-openshell-sandbox.sh‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -150,6 +150,21 @@ run_in_netnone() {
150150
podman run --rm --network none --entrypoint "" "$image" "$@"
151151
}
152152

153+
# Plugins enabled for Claude runs through the OpenShell backend: everything the
154+
# image ships except agent-eval-harness. That plugin's SessionStart hook
155+
# bulk-installs MLflow with uv, and uv reuses a policy-DNS synthetic IP past
156+
# its ~30s mapping lifetime. The resulting transparent_tcp_mapping_denied
157+
# while other relays are open crashes the OpenShell supervisor ("control-mode
158+
# proxy accept loop exited unexpectedly"), upstream NVIDIA/OpenShell#3396.
159+
# Drop this once the supervisor survives that denial.
160+
CLAUDE_E2E_PLUGINS="$(run_in "$CLAUDE_SANDBOX" python3 -c "
161+
import json, pathlib
162+
d = json.loads(pathlib.Path('/sandbox/.claude/settings.json').read_text())
163+
names = sorted({k.split('@')[0] for k in d.get('enabledPlugins', {})})
164+
print(','.join(n for n in names if n != 'agent-eval-harness'))
165+
")"
166+
print_step "Claude OpenShell runs use AGENT_ENABLED_PLUGINS=$CLAUDE_E2E_PLUGINS"
167+
153168
# --- shared sandbox checks ---
154169
print_header "=== shared sandbox: binaries ==="
155170

@@ -363,6 +378,7 @@ else
363378

364379
print_step "Running Claude Code via agentic-ci (openshell backend)..."
365380
RC=0
381+
AGENT_ENABLED_PLUGINS="$CLAUDE_E2E_PLUGINS" \
366382
agentic-ci run "Reply with only the word pong" \
367383
--backend openshell \
368384
--image "$CLAUDE_SANDBOX" \
@@ -438,6 +454,7 @@ else
438454
print_step "Running routed skill (openshell, claude-code, classifier)..."
439455
ROUTED_LOG="$TMPDIR_E2E/routed-claude.log"
440456
RC=0
457+
AGENT_ENABLED_PLUGINS="$CLAUDE_E2E_PLUGINS" \
441458
"$(agentic_python)" "$SCRIPT_DIR/routed_skill_driver.py" \
442459
--backend openshell --harness claude-code \
443460
--image "$CLAUDE_SANDBOX" --workdir "$WORKDIR" \
@@ -505,6 +522,7 @@ POLICY
505522
print_step "Running Claude Code with repo-level policy (packages.redhat.com allowed)..."
506523
POLICY_LOG="$TMPDIR_E2E/repo-policy.log"
507524
RC=0
525+
AGENT_ENABLED_PLUGINS="$CLAUDE_E2E_PLUGINS" \
508526
agentic-ci run \
509527
"Use curl to fetch https://packages.redhat.com. If you get a response, reply with only the word pong. If you cannot reach it, reply with only the word fail." \
510528
--backend openshell \
@@ -538,6 +556,7 @@ CONFIG
538556
print_step "Running Claude Code with setup steps..."
539557
SETUP_LOG="$TMPDIR_E2E/setup-steps.log"
540558
RC=0
559+
AGENT_ENABLED_PLUGINS="$CLAUDE_E2E_PLUGINS" \
541560
agentic-ci run \
542561
"Check if the file .setup-marker exists and contains 'setup-complete'. If yes, reply with only the word pong. If not, reply with only the word fail." \
543562
--backend openshell \
@@ -571,6 +590,7 @@ CONFIG
571590
SKIP_LOG="$TMPDIR_E2E/skip-setup.log"
572591
RC=0
573592
AGENTIC_CI_SKIP_SETUP=1 \
593+
AGENT_ENABLED_PLUGINS="$CLAUDE_E2E_PLUGINS" \
574594
agentic-ci run \
575595
"Check if the file .setup-marker exists. If yes, reply with only the word fail. If not, reply with only the word pong." \
576596
--backend openshell \
@@ -605,6 +625,7 @@ GITIGNORE
605625
print_step "Running Claude Code to create a file in a gitignored directory..."
606626
VERDICT_LOG="$TMPDIR_E2E/verdict-download.log"
607627
RC=0
628+
AGENT_ENABLED_PLUGINS="$CLAUDE_E2E_PLUGINS" \
608629
agentic-ci run \
609630
"Create the directory autofix-output/ then write the file autofix-output/verdict.json with the content {\"verdict\": \"committed\"}. Do not say anything else." \
610631
--backend openshell \

0 commit comments

Comments
 (0)